From d81f424cfaa0d35f9fbf6f2bc51164aefb9d231b Mon Sep 17 00:00:00 2001 From: kcreddy Date: Thu, 6 Aug 2026 19:09:46 +0530 Subject: [PATCH 1/3] ssi: add processor tags and preserve_original_event on failure (4/4) Add unique tags to every ingest pipeline processor and add `preserve_original_event` to pipeline-level `on_failure` handlers for the 15 GA packages owned by elastic/security-service-integrations. This is the final group of four PRs standardising all SSI-owned ingest pipelines. Updates #20558 Co-Authored-By: Claude Sonnet 4.6 --- packages/airlock_digital/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 10 +- .../elasticsearch/ingest_pipeline/default.yml | 7 + .../elasticsearch/ingest_pipeline/default.yml | 4 + packages/airlock_digital/manifest.yml | 2 +- packages/armis/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 9 +- .../elasticsearch/ingest_pipeline/default.yml | 39 ++++-- .../elasticsearch/ingest_pipeline/default.yml | 24 ++++ packages/armis/manifest.yml | 2 +- packages/bitsight/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 22 ++- packages/bitsight/manifest.yml | 2 +- packages/cyera/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 4 + .../elasticsearch/ingest_pipeline/default.yml | 19 +++ .../elasticsearch/ingest_pipeline/default.yml | 13 ++ .../elasticsearch/ingest_pipeline/default.yml | 10 +- packages/cyera/manifest.yml | 2 +- packages/entro/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 12 ++ packages/entro/manifest.yml | 2 +- packages/extrahop/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 12 +- .../elasticsearch/ingest_pipeline/default.yml | 13 ++ packages/extrahop/manifest.yml | 2 +- packages/neon_cyber/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 2 + .../elasticsearch/ingest_pipeline/default.yml | 4 +- packages/neon_cyber/manifest.yml | 2 +- packages/nozomi_networks/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 33 +++++ .../elasticsearch/ingest_pipeline/default.yml | 49 +++++++ .../elasticsearch/ingest_pipeline/default.yml | 9 +- .../elasticsearch/ingest_pipeline/default.yml | 9 +- .../elasticsearch/ingest_pipeline/default.yml | 53 +++++++ .../elasticsearch/ingest_pipeline/default.yml | 14 +- .../elasticsearch/ingest_pipeline/default.yml | 23 ++- .../elasticsearch/ingest_pipeline/default.yml | 22 +++ packages/nozomi_networks/manifest.yml | 2 +- packages/qualys_gav/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 131 ++++++++++++++++-- packages/qualys_gav/manifest.yml | 2 +- packages/qualys_was/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 62 +++++++-- .../pipeline_knowledge_base.yml | 23 ++- packages/qualys_was/manifest.yml | 2 +- packages/swimlane/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 42 +++++- .../elasticsearch/ingest_pipeline/default.yml | 46 +++++- .../elasticsearch/ingest_pipeline/default.yml | 46 +++++- .../elasticsearch/ingest_pipeline/default.yml | 48 ++++++- packages/swimlane/manifest.yml | 2 +- .../ti_cyware_intel_exchange/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 23 ++- .../ti_cyware_intel_exchange/manifest.yml | 2 +- packages/ti_greynoise/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 17 +++ packages/ti_greynoise/manifest.yml | 2 +- packages/vectra_rux/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 8 ++ .../elasticsearch/ingest_pipeline/default.yml | 11 +- .../elasticsearch/ingest_pipeline/default.yml | 12 +- .../elasticsearch/ingest_pipeline/default.yml | 48 +++++++ .../elasticsearch/ingest_pipeline/default.yml | 7 +- packages/vectra_rux/manifest.yml | 2 +- packages/withsecure_elements/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 7 +- .../elasticsearch/ingest_pipeline/default.yml | 8 +- packages/withsecure_elements/manifest.yml | 2 +- 71 files changed, 971 insertions(+), 94 deletions(-) diff --git a/packages/airlock_digital/changelog.yml b/packages/airlock_digital/changelog.yml index 68a4e2b5255..63c39d6e5bf 100644 --- a/packages/airlock_digital/changelog.yml +++ b/packages/airlock_digital/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.5.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "0.5.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/airlock_digital/data_stream/agent/elasticsearch/ingest_pipeline/default.yml b/packages/airlock_digital/data_stream/agent/elasticsearch/ingest_pipeline/default.yml index c5a3b516c4c..a020583dbc9 100644 --- a/packages/airlock_digital/data_stream/agent/elasticsearch/ingest_pipeline/default.yml +++ b/packages/airlock_digital/data_stream/agent/elasticsearch/ingest_pipeline/default.yml @@ -92,6 +92,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ed15c765 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -129,6 +130,7 @@ processors: if: ctx.json?.ip != '' on_failure: - append: + tag: append_error_message_b1bdbb98 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -152,6 +154,7 @@ processors: if: ctx.json?.lastcheckin != null && ctx.json.lastcheckin != '' on_failure: - append: + tag: append_error_message_1155faed field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -163,6 +166,7 @@ processors: if: ctx.json?.localip != '' on_failure: - append: + tag: append_error_message_d86b373c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -277,7 +281,7 @@ processors: value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: field: json.policy_details.commlist - tag: foreach_policy_details_commlist_ip + tag: foreach_json_policy_details_commlist_761aaae4 if: ctx.json?.policy_details?.commlist instanceof List processor: append: @@ -462,6 +466,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_80c7c540 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -540,6 +545,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_97fa019b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -583,6 +589,7 @@ processors: type: string ignore_missing: true - script: + tag: script_1fd0624f description: Map status to corresponding agent status. lang: painless params: @@ -681,6 +688,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/airlock_digital/data_stream/execution_histories/elasticsearch/ingest_pipeline/default.yml b/packages/airlock_digital/data_stream/execution_histories/elasticsearch/ingest_pipeline/default.yml index 2bf82ca53fc..28f6711370b 100644 --- a/packages/airlock_digital/data_stream/execution_histories/elasticsearch/ingest_pipeline/default.yml +++ b/packages/airlock_digital/data_stream/execution_histories/elasticsearch/ingest_pipeline/default.yml @@ -38,6 +38,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -81,6 +82,7 @@ processors: if: ctx.json?.datetime != null && ctx.json.datetime != '' on_failure: - append: + tag: append_error_message_bf22f8e7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -99,11 +101,13 @@ processors: copy_from: airlock_digital.execution_histories.filename ignore_empty_value: true - gsub: + tag: gsub_file_path_517235af field: file.path pattern: '\\' replacement: '/' ignore_failure: true - split: + tag: split_file_path_to_file_name_19cfdeb5 field: file.path separator: '/' target_field: file.name @@ -265,9 +269,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d064192d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: + tag: script_5a14c280 description: Map type to corresponding execution type. lang: painless params: @@ -364,6 +370,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/airlock_digital/data_stream/server_activities/elasticsearch/ingest_pipeline/default.yml b/packages/airlock_digital/data_stream/server_activities/elasticsearch/ingest_pipeline/default.yml index 64ff27bf710..74313a2d64a 100644 --- a/packages/airlock_digital/data_stream/server_activities/elasticsearch/ingest_pipeline/default.yml +++ b/packages/airlock_digital/data_stream/server_activities/elasticsearch/ingest_pipeline/default.yml @@ -68,6 +68,7 @@ processors: if: ctx.json?.datetime != null && ctx.json.datetime != '' on_failure: - append: + tag: append_error_message_63229d47 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -107,6 +108,7 @@ processors: if: ctx.event?.action != '' on_failure: - append: + tag: append_error_message_ce1e3af4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - join: @@ -116,6 +118,7 @@ processors: if: ctx.event?.action != '' on_failure: - append: + tag: append_error_message_abbd6eec field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -179,6 +182,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/airlock_digital/manifest.yml b/packages/airlock_digital/manifest.yml index 1edfa9666dc..ed297975e7a 100644 --- a/packages/airlock_digital/manifest.yml +++ b/packages/airlock_digital/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: airlock_digital title: Airlock Digital -version: 0.5.1 +version: 0.5.2 description: Collect logs from Airlock Digital with Elastic Agent. type: integration categories: diff --git a/packages/armis/changelog.yml b/packages/armis/changelog.yml index ce1b77a11b1..e018ace14fe 100644 --- a/packages/armis/changelog.yml +++ b/packages/armis/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.6.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "0.6.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/armis/data_stream/alert/elasticsearch/ingest_pipeline/default.yml b/packages/armis/data_stream/alert/elasticsearch/ingest_pipeline/default.yml index e081430d556..3721f45101d 100755 --- a/packages/armis/data_stream/alert/elasticsearch/ingest_pipeline/default.yml +++ b/packages/armis/data_stream/alert/elasticsearch/ingest_pipeline/default.yml @@ -42,6 +42,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - fingerprint: @@ -76,6 +77,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_6bf4e364 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -132,7 +134,7 @@ processors: allow_duplicates: false - foreach: field: armis.alert.device_ids - tag: foreach_deviceIds + tag: foreach_armis_alert_device_ids_4938258b if: ctx.armis?.alert?.device_ids instanceof List processor: append: @@ -151,6 +153,7 @@ processors: if: ctx.json?.lastAlertUpdateTime != null && ctx.json.lastAlertUpdateTime != '' on_failure: - append: + tag: append_error_message_962c6444 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -218,6 +221,7 @@ processors: } on_failure: - append: + tag: append_error_message_b608e76d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -241,6 +245,7 @@ processors: if: ctx.json?.statusChangeTime != null && ctx.json.statusChangeTime != '' on_failure: - append: + tag: append_error_message_9981e61c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -259,6 +264,7 @@ processors: if: ctx.json?.time != null && ctx.json.time != '' on_failure: - append: + tag: append_error_message_7ab96520 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -319,6 +325,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/armis/data_stream/device/elasticsearch/ingest_pipeline/default.yml b/packages/armis/data_stream/device/elasticsearch/ingest_pipeline/default.yml index 6c4a0c4b27b..70c71029e4d 100644 --- a/packages/armis/data_stream/device/elasticsearch/ingest_pipeline/default.yml +++ b/packages/armis/data_stream/device/elasticsearch/ingest_pipeline/default.yml @@ -42,6 +42,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - fingerprint: @@ -120,7 +121,7 @@ processors: ignore_missing: true - foreach: field: json.dataSources - tag: foreach_dataSources + tag: foreach_json_dataSources_44a528e5 if: ctx.json?.dataSources instanceof List processor: foreach: @@ -141,7 +142,7 @@ processors: ignore_missing: true - foreach: field: json.dataSources - tag: foreach_dataSources + tag: foreach_json_dataSources_80aabde7 if: ctx.json?.dataSources instanceof List processor: foreach: @@ -162,7 +163,7 @@ processors: ignore_missing: true - foreach: field: json.dataSources - tag: foreach_dataSources + tag: foreach_json_dataSources_b14257f2 if: ctx.json?.dataSources instanceof List processor: date: @@ -179,7 +180,7 @@ processors: ignore_missing: true - foreach: field: json.dataSources - tag: foreach_dataSources + tag: foreach_json_dataSources_23b1bbea if: ctx.json?.dataSources instanceof List processor: foreach: @@ -194,7 +195,7 @@ processors: ignore_missing: true - foreach: field: json.dataSources - tag: foreach_dataSources + tag: foreach_json_dataSources_17aa49c5 if: ctx.json?.dataSources instanceof List processor: remove: @@ -224,11 +225,12 @@ processors: if: ctx.json?.firstSeen != null && ctx.json.firstSeen != '' on_failure: - append: + tag: append_error_message_3f050535 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.start - tag: set_event_start_from_device_first_seen + tag: set_event_start_62b39146 copy_from: armis.device.first_seen ignore_empty_value: true - convert: @@ -251,6 +253,7 @@ processors: if: ctx.json?.ipAddress instanceof String && ctx.json.ipAddress != '' on_failure: - append: + tag: append_error_message_1ef97379 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -272,7 +275,7 @@ processors: value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: field: armis.device.ip_address - tag: foreach_armis_device_ip_address + tag: foreach_armis_device_ip_address_83fc270f if: ctx.armis?.device?.ip_address instanceof List processor: append: @@ -282,7 +285,7 @@ processors: allow_duplicates: false - foreach: field: armis.device.ip_address - tag: foreach_armis_device_ip_address + tag: foreach_armis_device_ip_address_cbd753d5 if: ctx.armis?.device?.ip_address instanceof List processor: append: @@ -333,6 +336,7 @@ processors: if: ctx.json?.lastSeen != null && ctx.json.lastSeen != '' on_failure: - append: + tag: append_error_message_36ac071d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -349,9 +353,11 @@ processors: if: ctx.json?.macAddress instanceof String && ctx.json.macAddress != '' on_failure: - append: + tag: append_error_message_304849a2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_armis_device_mac_address_886e7aa3 field: armis.device.mac_address if: ctx.armis?.device?.mac_address instanceof List ignore_failure: true @@ -362,6 +368,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - foreach: + tag: foreach_host_mac_cf1efd0d field: host.mac if: ctx.host?.mac instanceof List ignore_failure: true @@ -377,6 +384,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_host_mac_15ece91a field: host.mac if: ctx.host?.mac instanceof List ignore_failure: true @@ -419,9 +427,11 @@ processors: if: ctx.json?.names instanceof String && ctx.json.names != '' on_failure: - append: + tag: append_error_message_76b1b9d9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_armis_device_names_01a73f31 field: armis.device.names if: ctx.armis?.device?.names instanceof List ignore_failure: true @@ -441,6 +451,7 @@ processors: tag: lowercase_host_name ignore_missing: true - foreach: + tag: foreach_host_name_13da92ce field: host.name if: ctx.host?.name instanceof List ignore_failure: true @@ -498,7 +509,7 @@ processors: ignore_missing: true - foreach: field: json.protections - tag: foreach_json_protections + tag: foreach_json_protections_ab2da63a if: ctx.json?.protections instanceof List processor: convert: @@ -509,7 +520,7 @@ processors: ignore_missing: true - foreach: field: json.protections - tag: foreach_json_protections + tag: foreach_json_protections_7ad79d22 if: ctx.json?.protections instanceof List processor: date: @@ -525,6 +536,7 @@ processors: field: _ingest._value.lastSeenTime ignore_missing: true - foreach: + tag: foreach_json_protections_40ae4388 field: json.protections if: ctx.json?.protections instanceof List processor: @@ -534,6 +546,7 @@ processors: target_field: _ingest._value.protection_name ignore_missing: true - foreach: + tag: foreach_json_protections_dc8b0544 field: json.protections if: ctx.json?.protections instanceof List processor: @@ -557,6 +570,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_750fa912 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -567,6 +581,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e1222968 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -581,6 +596,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_3141b072 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -636,7 +652,7 @@ processors: allow_duplicates: false - foreach: field: armis.device.user_ids - tag: foreach_armis_device_user_ids + tag: foreach_armis_device_user_ids_eb2fdd9d if: ctx.armis?.device?.user_ids instanceof List processor: append: @@ -702,6 +718,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/armis/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml b/packages/armis/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml index 928e26d37c5..99e28bd2e7c 100755 --- a/packages/armis/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml +++ b/packages/armis/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml @@ -42,6 +42,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - fingerprint: @@ -85,6 +86,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5e599058 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -145,6 +147,7 @@ processors: if: ctx.json?.avmRatingManualUpdateTime != null && ctx.json.avmRatingManualUpdateTime != '' on_failure: - append: + tag: append_error_message_0aad49ae field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -163,6 +166,7 @@ processors: if: ctx.json?.cisaDueDate != null && ctx.json.cisaDueDate != '' on_failure: - append: + tag: append_error_message_1acd96cf field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -188,6 +192,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_96955690 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -218,6 +223,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_669a5f43 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -228,6 +234,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f02e1804 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -238,6 +245,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2b6ef430 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -251,6 +259,7 @@ processors: if: ctx.json?.firstDetected != null && ctx.json.firstDetected != '' on_failure: - append: + tag: append_error_message_5e7ac6c4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -269,6 +278,7 @@ processors: if: ctx.json?.firstReferencePublishDate != null && ctx.json.firstReferencePublishDate != '' on_failure: - append: + tag: append_error_message_dcf2b6dc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -282,6 +292,7 @@ processors: if: ctx.json?.firstWeaponizedReferencePublishDate != null && ctx.json.firstWeaponizedReferencePublishDate != '' on_failure: - append: + tag: append_error_message_58d0c6dd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -292,6 +303,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_cb6fcfb8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -317,6 +329,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_31efb147 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -332,6 +345,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4e51ce63 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -345,6 +359,7 @@ processors: if: ctx.json?.lastDetected != null && ctx.json.lastDetected != '' on_failure: - append: + tag: append_error_message_8d9aba08 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -358,6 +373,7 @@ processors: if: ctx.json?.latestExploitUpdate != null && ctx.json.latestExploitUpdate != '' on_failure: - append: + tag: append_error_message_593cce52 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -368,6 +384,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8ab38101 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -378,6 +395,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9ab7e23d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -396,6 +414,7 @@ processors: if: ctx.json?.publishedDate != null && ctx.json.publishedDate != '' on_failure: - append: + tag: append_error_message_02b400b2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -406,6 +425,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_3e72b7f2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -421,6 +441,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_10fe04c2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -516,6 +537,7 @@ processors: if: ctx.json?.vulnerability_match?.firstDetected != null && ctx.json.vulnerability_match.firstDetected != '' on_failure: - append: + tag: append_error_message_ddecb674 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -534,6 +556,7 @@ processors: if: ctx.json?.vulnerability_match?.lastDetected != null && ctx.json.vulnerability_match.lastDetected != '' on_failure: - append: + tag: append_error_message_1036c2f4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -625,6 +648,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/armis/manifest.yml b/packages/armis/manifest.yml index 2a1d299839d..e08ed30e8f1 100644 --- a/packages/armis/manifest.yml +++ b/packages/armis/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: armis title: Armis -version: "0.6.1" +version: "0.6.2" description: Collect logs from Armis with Elastic Agent. type: integration categories: diff --git a/packages/bitsight/changelog.yml b/packages/bitsight/changelog.yml index 33760ef5a7e..b331b77d613 100644 --- a/packages/bitsight/changelog.yml +++ b/packages/bitsight/changelog.yml @@ -1,3 +1,8 @@ +- version: "0.4.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "0.4.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/bitsight/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml b/packages/bitsight/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml index 9e62bad9151..5061be5545f 100644 --- a/packages/bitsight/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml +++ b/packages/bitsight/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml @@ -3,6 +3,7 @@ description: Pipeline for processing Bitsight Vulnerability output processors: # Drop retry continuation events early (before any processing) - drop: + tag: drop_7397da51 if: ctx.message == 'retry' - set: @@ -36,24 +37,29 @@ processors: if: ctx.event?.original instanceof String on_failure: - append: + tag: append_error_message_70ba1955 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' # Remove event.original unless preserve_original_event tag is present - remove: + tag: remove_event_original_86d11a95 field: event.original ignore_missing: true if: ctx.tags == null || !ctx.tags.contains('preserve_original_event') # Set ECS event fields for vulnerability events - set: + tag: set_event_kind_de80643c field: event.kind value: event - append: + tag: append_event_category_914f8241 field: event.category value: vulnerability allow_duplicates: false - append: + tag: append_event_type_3f43a39d field: event.type value: info allow_duplicates: false @@ -65,12 +71,14 @@ processors: target_field: "_id" on_failure: - append: + tag: append_error_message_39122456 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - ##################### Scripts for processing input ##################### + ##################### Scripts for processing input ##################### - script: + tag: script_924392cb description: Drops null/empty values recursively. lang: painless source: | @@ -89,23 +97,27 @@ processors: dropEmptyFields(ctx); - date: + tag: date_bitsight_threat_last_seen_date_to_timestamp_67a83afb field: bitsight.threat.last_seen_date formats: ["yyyy-MM-dd"] target_field: '@timestamp' ignore_failure: true - set: + tag: set_vulnerability_id_bc0a7786 field: vulnerability.id copy_from: bitsight.threat.name ignore_failure: true - set: + tag: set_vulnerability_scanner_vendor_0cbe0ad1 field: vulnerability.scanner.vendor value: "Bitsight" ignore_failure: true # Map Bitsight severity.level → ECS severity - script: + tag: script_2e28b975 lang: painless source: | if (ctx.bitsight?.threat?.severity?.level != null) { @@ -126,36 +138,42 @@ processors: # Extract score from severity.details field - grok: + tag: grok_bitsight_threat_severity_details_debdc64f field: bitsight.threat.severity.details patterns: ["CVSS %{NUMBER:vulnerability.score.base:float}"] ignore_failure: true # Extract IP from evidence identifier field (e.g., 1.2.3.4:443) - remove: + tag: remove_host_1655f80e field: host ignore_failure: true - grok: + tag: grok_bitsight_evidence_identifier_3419039e field: bitsight.evidence.identifier patterns: ["%{IP:extracted_ip}:%{NUMBER}"] ignore_failure: true - append: + tag: append_host_ip_b2dd9500 field: host.ip value: "{{{extracted_ip}}}" if: ctx.extracted_ip != null ignore_failure: true allow_duplicates: false - remove: + tag: remove_extracted_ip_860987b0 field: extracted_ip ignore_failure: true -#################### Error Log fields #################### + #################### Error Log fields #################### - set: field: event.kind tag: set_pipeline_error_into_event_kind value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/bitsight/manifest.yml b/packages/bitsight/manifest.yml index 887d6939744..aece0dff683 100644 --- a/packages/bitsight/manifest.yml +++ b/packages/bitsight/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: bitsight title: "Bitsight" -version: 0.4.1 +version: 0.4.2 source: license: "Elastic-2.0" description: "Ingest data from the Bitsight API." diff --git a/packages/cyera/changelog.yml b/packages/cyera/changelog.yml index 54fa55e6a50..3a0f091a2dc 100644 --- a/packages/cyera/changelog.yml +++ b/packages/cyera/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.10.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "0.10.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/cyera/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/cyera/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 76cd16ad273..ee80c8c9aba 100644 --- a/packages/cyera/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cyera/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -38,6 +38,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -59,6 +60,7 @@ processors: if: ctx.json?.createdAt != null && ctx.json.createdAt != '' on_failure: - append: + tag: append_error_message_79f76e15 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -70,6 +72,7 @@ processors: if: ctx.json?.updatedAt != null && ctx.json.updatedAt != '' on_failure: - append: + tag: append_error_message_070d79fb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -101,6 +104,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a364938a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -298,6 +302,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/cyera/data_stream/classification/elasticsearch/ingest_pipeline/default.yml b/packages/cyera/data_stream/classification/elasticsearch/ingest_pipeline/default.yml index cec47af5506..17612e8c024 100644 --- a/packages/cyera/data_stream/classification/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cyera/data_stream/classification/elasticsearch/ingest_pipeline/default.yml @@ -86,6 +86,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ebd0f3e3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -107,6 +108,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9c6696b1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -157,6 +159,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_033e2e4f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -232,6 +235,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/cyera/data_stream/datastore/elasticsearch/ingest_pipeline/default.yml b/packages/cyera/data_stream/datastore/elasticsearch/ingest_pipeline/default.yml index 99df7b9dd4a..c1e59c85ade 100644 --- a/packages/cyera/data_stream/datastore/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cyera/data_stream/datastore/elasticsearch/ingest_pipeline/default.yml @@ -2,6 +2,7 @@ description: Pipeline for processing Datastore logs. processors: - drop: + tag: drop_8a0518c5 description: Ignore retry placeholder message. if: ctx.message == 'retry' - remove: @@ -84,6 +85,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_45d7036a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -125,6 +127,7 @@ processors: if: ctx.json?.createdDate != null && ctx.json.createdDate != '' on_failure: - append: + tag: append_error_message_e3b068e9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -195,6 +198,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_0efb3603 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -206,6 +210,7 @@ processors: if: ctx.json?.discoveredDate != null && ctx.json.discoveredDate != '' on_failure: - append: + tag: append_error_message_cd2ce22b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -221,6 +226,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4f35a635 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -241,6 +247,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_3bdea7e6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -261,6 +268,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d8f829ba field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -271,6 +279,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_32971783 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -281,6 +290,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ce35ec72 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -292,6 +302,7 @@ processors: if: ctx.json?.lastDataRefresh != null && ctx.json.lastDataRefresh != '' on_failure: - append: + tag: append_error_message_269ab3c0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -303,6 +314,7 @@ processors: if: ctx.json?.lastModifiedTime != null && ctx.json.lastModifiedTime != '' on_failure: - append: + tag: append_error_message_4c9672f7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -324,6 +336,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_69afca3a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -407,6 +420,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_aac85816 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -417,6 +431,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_041de9d5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -427,6 +442,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_93410457 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -437,6 +453,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_917f772d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -447,6 +464,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_34ec9088 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -597,6 +615,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/cyera/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/cyera/data_stream/event/elasticsearch/ingest_pipeline/default.yml index fa6e6072dd1..f4bdd47fadb 100644 --- a/packages/cyera/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cyera/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -38,6 +38,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -135,6 +136,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_46ec7dbe field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -145,6 +147,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4d974a04 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -155,6 +158,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_0ada6d1e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -229,6 +233,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c37c421d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -260,6 +265,7 @@ processors: if: ctx.json?.date != null && ctx.json.date != '' on_failure: - append: + tag: append_error_message_a2c9ac5b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -291,6 +297,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_994f3e70 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -301,6 +308,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a15a8f4f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -321,6 +329,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_573c6c2d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -331,6 +340,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ac684c14 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -341,6 +351,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_b972221c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -536,6 +547,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c052db5e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -657,6 +669,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/cyera/data_stream/issue/elasticsearch/ingest_pipeline/default.yml b/packages/cyera/data_stream/issue/elasticsearch/ingest_pipeline/default.yml index 2d7bc131c49..3394e628f04 100644 --- a/packages/cyera/data_stream/issue/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cyera/data_stream/issue/elasticsearch/ingest_pipeline/default.yml @@ -85,6 +85,7 @@ processors: if: ctx.json?.createdDate != null && ctx.json.createdDate != '' on_failure: - append: + tag: append_error_message_3e918fdb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -100,6 +101,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_deeefde3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -115,6 +117,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_cf579102 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -125,6 +128,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5bf6fb88 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -149,7 +153,7 @@ processors: ignore_missing: true - foreach: field: json.datastoreOwners - tag: foreach_datastoreOwners_datastoreOwnerUid + tag: foreach_json_datastoreOwners_b7acef9e if: ctx.json?.datastoreOwners instanceof List processor: append: @@ -179,7 +183,7 @@ processors: ignore_missing: true - foreach: field: json.datastoreOwners - tag: foreach_datastoreOwners_ownerType + tag: foreach_json_datastoreOwners_920ad464 if: ctx.json?.datastoreOwners instanceof List processor: append: @@ -411,6 +415,7 @@ processors: if: ctx.json?.updatedDate != null && ctx.json.updatedDate != '' on_failure: - append: + tag: append_error_message_0fd23ee0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -478,6 +483,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/cyera/manifest.yml b/packages/cyera/manifest.yml index 0386e3049db..1bd9bb4c130 100644 --- a/packages/cyera/manifest.yml +++ b/packages/cyera/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: cyera title: Cyera -version: 0.10.1 +version: 0.10.2 description: Collect logs from Cyera with Elastic Agent. type: integration categories: diff --git a/packages/entro/changelog.yml b/packages/entro/changelog.yml index dcc1bbd8c46..00e6f370659 100644 --- a/packages/entro/changelog.yml +++ b/packages/entro/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.4.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "0.4.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/entro/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/entro/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 0384aa9cb69..ad6d3077b6d 100644 --- a/packages/entro/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/entro/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -19,8 +19,10 @@ processors: if: ctx.date != null on_failure: - remove: + tag: remove_date_1a15cc9b field: date - append: + tag: append_error_message_bf1997ac field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -36,39 +38,49 @@ processors: target_field: _id ignore_missing: true - set: + tag: set_event_id_7cf3f855 field: event.id copy_from: request_id ignore_empty_value: true - set: + tag: set_vulnerability_value_d873757a field: vulnerability.value copy_from: exposed_value ignore_empty_value: true - set: + tag: set_log_origin_file_line_d1fd2935 field: log.origin.file.line copy_from: line_number ignore_empty_value: true - set: + tag: set_vulnerability_category_31985606 field: vulnerability.category value: ["Secret"] ignore_empty_value: true if: ctx.secret_type != null - set: + tag: set_vulnerability_description_a3826a5e field: vulnerability.description copy_from: secret_type ignore_empty_value: true - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_category_13b30ef2 field: event.category value: ["vulnerability"] - set: + tag: set_event_type_ec95f7f2 field: event.type value: ["info"] - set: + tag: set_event_dataset_d3e82901 field: event.dataset value: "entro.audit" - remove: + tag: remove_07170528 field: - date - request_id diff --git a/packages/entro/manifest.yml b/packages/entro/manifest.yml index ad4ecdfaf1e..ceb902134b2 100644 --- a/packages/entro/manifest.yml +++ b/packages/entro/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.5 name: entro title: "Entro" -version: 0.4.1 +version: 0.4.2 description: "Collect logs from Entro with Elastic Agent." type: integration categories: diff --git a/packages/extrahop/changelog.yml b/packages/extrahop/changelog.yml index 17fe036491d..a18c4054ab8 100644 --- a/packages/extrahop/changelog.yml +++ b/packages/extrahop/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.3.3" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "0.3.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/extrahop/data_stream/detection/elasticsearch/ingest_pipeline/default.yml b/packages/extrahop/data_stream/detection/elasticsearch/ingest_pipeline/default.yml index ae1ceac6a08..a3529abb2ff 100644 --- a/packages/extrahop/data_stream/detection/elasticsearch/ingest_pipeline/default.yml +++ b/packages/extrahop/data_stream/detection/elasticsearch/ingest_pipeline/default.yml @@ -97,6 +97,7 @@ processors: if: ctx.json?.create_time != null && ctx.json.create_time != '' on_failure: - append: + tag: append_error_message_b79de794 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -133,6 +134,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_35ae4b39 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -214,6 +216,7 @@ processors: if: ctx.json?.mod_time != null && ctx.json.mod_time != '' on_failure: - append: + tag: append_error_message_99c360ce field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -306,7 +309,7 @@ processors: value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: field: json.participants - tag: foreach_participants_object_value + tag: foreach_json_participants_a474cd8b if: ctx.json?.participants instanceof List processor: append: @@ -342,6 +345,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_15c50513 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -362,6 +366,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9419db17 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -377,6 +382,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5ba0a5d7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -397,6 +403,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_25b11484 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -413,6 +420,7 @@ processors: if: ctx.json?.start_time != null && ctx.json.start_time != '' on_failure: - append: + tag: append_error_message_268c6652 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -460,6 +468,7 @@ processors: if: ctx.json?.update_time != null && ctx.json.update_time != '' on_failure: - append: + tag: append_error_message_424508fd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -550,6 +559,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/extrahop/data_stream/investigation/elasticsearch/ingest_pipeline/default.yml b/packages/extrahop/data_stream/investigation/elasticsearch/ingest_pipeline/default.yml index 2cc2834ad4f..2d0bdd106e1 100644 --- a/packages/extrahop/data_stream/investigation/elasticsearch/ingest_pipeline/default.yml +++ b/packages/extrahop/data_stream/investigation/elasticsearch/ingest_pipeline/default.yml @@ -73,9 +73,11 @@ processors: if: ctx.extrahop?.investigation.creation_time != null && ctx.extrahop.investigation.creation_time != '' on_failure: - remove: + tag: remove_extrahop_investigation_creation_time_d079ba15 field: extrahop.investigation.creation_time ignore_missing: true - append: + tag: append_error_message_c14230aa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -99,9 +101,11 @@ processors: if: ctx.extrahop?.investigation.end_time != null && ctx.extrahop.investigation.end_time != '' on_failure: - remove: + tag: remove_extrahop_investigation_end_time_2a782657 field: extrahop.investigation.end_time ignore_missing: true - append: + tag: append_error_message_c69749f0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -136,6 +140,7 @@ processors: ctx.event.id = jsonId + '-' + epochMillis; on_failure: - append: + tag: append_error_message_3542463b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: @@ -150,6 +155,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_986710ed field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -167,9 +173,11 @@ processors: if: ctx.extrahop?.investigation.last_interaction_time != null && ctx.extrahop.investigation.last_interaction_time != '' on_failure: - remove: + tag: remove_extrahop_investigation_last_interaction_time_a52688f7 field: extrahop.investigation.last_interaction_time ignore_missing: true - append: + tag: append_error_message_19da6556 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -186,9 +194,11 @@ processors: if: ctx.extrahop?.investigation.start_time != null && ctx.extrahop.investigation.start_time != '' on_failure: - remove: + tag: remove_extrahop_investigation_start_time_578b29b8 field: extrahop.investigation.start_time ignore_missing: true - append: + tag: append_error_message_7d54302c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -214,9 +224,11 @@ processors: if: ctx.extrahop?.investigation.update_time != null && ctx.extrahop.investigation.update_time != '' on_failure: - remove: + tag: remove_extrahop_investigation_update_time_bc6d79a5 field: extrahop.investigation.update_time ignore_missing: true - append: + tag: append_error_message_0f683842 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -266,6 +278,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/extrahop/manifest.yml b/packages/extrahop/manifest.yml index 919a816edc6..bba77dd5303 100644 --- a/packages/extrahop/manifest.yml +++ b/packages/extrahop/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: extrahop title: ExtraHop -version: 0.3.2 +version: 0.3.3 description: Collect logs from ExtraHop RevealX 360 with Elastic Agent. type: integration categories: diff --git a/packages/neon_cyber/changelog.yml b/packages/neon_cyber/changelog.yml index 67baff87e47..82a8b06a73e 100644 --- a/packages/neon_cyber/changelog.yml +++ b/packages/neon_cyber/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.2.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "0.2.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/neon_cyber/data_stream/detections/elasticsearch/ingest_pipeline/default.yml b/packages/neon_cyber/data_stream/detections/elasticsearch/ingest_pipeline/default.yml index 0879e156126..cd1e92c1088 100644 --- a/packages/neon_cyber/data_stream/detections/elasticsearch/ingest_pipeline/default.yml +++ b/packages/neon_cyber/data_stream/detections/elasticsearch/ingest_pipeline/default.yml @@ -63,6 +63,7 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_bd48fdd8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -73,6 +74,7 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_33d49aba field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: diff --git a/packages/neon_cyber/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/neon_cyber/data_stream/events/elasticsearch/ingest_pipeline/default.yml index 38d6d60e6b2..a9829c2c021 100644 --- a/packages/neon_cyber/data_stream/events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/neon_cyber/data_stream/events/elasticsearch/ingest_pipeline/default.yml @@ -63,16 +63,18 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_aecb25a8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: field: json.event_timestamp target_field: '@timestamp' - tag: date_event_timestamp + tag: date_json_event_timestamp_to_timestamp_40752292 formats: - ISO8601 on_failure: - append: + tag: append_error_message_6164d83c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: diff --git a/packages/neon_cyber/manifest.yml b/packages/neon_cyber/manifest.yml index 30044068415..c4eb7efd70e 100644 --- a/packages/neon_cyber/manifest.yml +++ b/packages/neon_cyber/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.5.0 name: neon_cyber title: 'Neon Cyber' -version: 0.2.1 +version: 0.2.2 source: license: 'Elastic-2.0' description: 'The Neon Cyber integration for the Elastic Stack' diff --git a/packages/nozomi_networks/changelog.yml b/packages/nozomi_networks/changelog.yml index 850b11eedab..c3a0e7772c1 100644 --- a/packages/nozomi_networks/changelog.yml +++ b/packages/nozomi_networks/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.3.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "0.3.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/nozomi_networks/data_stream/alert/elasticsearch/ingest_pipeline/default.yml b/packages/nozomi_networks/data_stream/alert/elasticsearch/ingest_pipeline/default.yml index 7c6cc51da21..41ea8d6f546 100644 --- a/packages/nozomi_networks/data_stream/alert/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nozomi_networks/data_stream/alert/elasticsearch/ingest_pipeline/default.yml @@ -39,6 +39,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -62,6 +63,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5c2b6b95 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -98,6 +100,7 @@ processors: if: ctx.json?.appliance_ip != '' on_failure: - append: + tag: append_error_message_d7422d5f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -141,6 +144,7 @@ processors: if: ctx.json?.closed_time != null && ctx.json.closed_time != 0 on_failure: - append: + tag: append_error_message_e36cb724 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -152,6 +156,7 @@ processors: if: ctx.json?.created_time != null on_failure: - append: + tag: append_error_message_09dfb237 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -198,6 +203,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_af2c512f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -244,10 +250,12 @@ processors: copy_from: nozomi_networks.alert.destination_ip ignore_empty_value: true - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -290,10 +298,12 @@ processors: copy_from: nozomi_networks.alert.source_ip ignore_empty_value: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -325,6 +335,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_90f56253 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -335,6 +346,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ee62e357 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -368,6 +380,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_15a1da31 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - uppercase: @@ -377,6 +390,7 @@ processors: if: ctx.destination?.mac != '' on_failure: - append: + tag: append_error_message_9c70d5b1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -393,6 +407,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_7d3d9226 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - uppercase: @@ -402,6 +417,7 @@ processors: if: ctx.source?.mac != '' on_failure: - append: + tag: append_error_message_bdc871d0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -437,6 +453,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c18fd4e5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -452,6 +469,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_42e49728 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -467,6 +485,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_3a78e55b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -482,6 +501,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_27d60d07 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -492,6 +512,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_3f70bf6c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -502,6 +523,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_877cad46 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -512,6 +534,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_0a0eee5e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -522,6 +545,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_0a9f77b5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -532,6 +556,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f445747f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -592,6 +617,7 @@ processors: if: ctx.json?.record_created_at != null on_failure: - append: + tag: append_error_message_0bc498e7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -607,6 +633,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8dc7a83d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -617,6 +644,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e9612755 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -632,6 +660,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_b064be8e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -647,6 +676,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_331466bd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -690,6 +720,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f7963078 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -711,6 +742,7 @@ processors: if: ctx.json?.time != null on_failure: - append: + tag: append_error_message_8cc9de5b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -836,6 +868,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/nozomi_networks/data_stream/asset/elasticsearch/ingest_pipeline/default.yml b/packages/nozomi_networks/data_stream/asset/elasticsearch/ingest_pipeline/default.yml index fd5b95e2028..ecd9a11f4d2 100644 --- a/packages/nozomi_networks/data_stream/asset/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nozomi_networks/data_stream/asset/elasticsearch/ingest_pipeline/default.yml @@ -39,6 +39,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -114,6 +115,7 @@ processors: if: ctx.json?.created_at != null on_failure: - append: + tag: append_error_message_38bf5935 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -145,6 +147,7 @@ processors: if: ctx.json?.end_of_sale_date != null on_failure: - append: + tag: append_error_message_1bc4e565 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -161,6 +164,7 @@ processors: if: ctx.json?.end_of_support_date != null on_failure: - append: + tag: append_error_message_ddcc5931 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -186,6 +190,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_b3ece5b1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -247,6 +252,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8a1b4c60 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -257,6 +263,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9863ca62 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -267,6 +274,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_13d2fac5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -277,6 +285,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_047e3765 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -288,6 +297,7 @@ processors: if: ctx.json?.last_activity_time != null on_failure: - append: + tag: append_error_message_243cc535 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -330,6 +340,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2a43a51c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -340,6 +351,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_eda36d6f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -356,6 +368,7 @@ processors: ctx.host.geo.location = locationList; on_failure: - append: + tag: append_error_message_d15cf3f9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -377,6 +390,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c4e630a3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - uppercase: @@ -385,6 +399,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f58efe9c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -435,6 +450,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_575e817c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -506,6 +522,7 @@ processors: if: ctx.json?.record_created_at != null on_failure: - append: + tag: append_error_message_f1a97165 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -521,6 +538,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_09f36983 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -536,6 +554,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_fa103412 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -546,6 +565,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_b2b240df field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -556,6 +576,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4040d2ef field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -566,6 +587,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_53a97965 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -576,6 +598,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_32e48a10 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -586,6 +609,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4f99c624 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -596,6 +620,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_6f950d6b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -606,6 +631,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_af26f970 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -616,6 +642,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_faa2b171 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -626,6 +653,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a8449afe field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -636,6 +664,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a4cd558c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -646,6 +675,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_612c300c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -656,6 +686,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_b2d994f6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -666,6 +697,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e8ad85d9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -676,6 +708,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_3c2e2a76 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -686,6 +719,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_210d965d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -696,6 +730,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c8f3407f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -706,6 +741,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a2e2e320 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -716,6 +752,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_abfdb1d0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -726,6 +763,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_6b4541de field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -736,6 +774,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a9ecf72f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -746,6 +785,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_361d6329 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -756,6 +796,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a36cd84a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -766,6 +807,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ca9b6956 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -776,6 +818,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_1ee2611f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -786,6 +829,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4c5120be field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -801,6 +845,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_31d18f8f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -816,6 +861,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d3e25e76 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -826,6 +872,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_0ba34c2e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -872,6 +919,7 @@ processors: if: ctx.json?.time != null on_failure: - append: + tag: append_error_message_b681cb9d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -966,6 +1014,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/nozomi_networks/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/nozomi_networks/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 642ca7b3dda..4737fa34067 100644 --- a/packages/nozomi_networks/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nozomi_networks/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -39,6 +39,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -76,6 +77,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_01b68fd4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -84,6 +86,7 @@ processors: target_field: nozomi_networks.audit.browser ignore_missing: true - user_agent: + tag: user_agent_nozomi_networks_audit_browser_aa9d400c field: nozomi_networks.audit.browser ignore_missing: true - set: @@ -131,6 +134,7 @@ processors: if: ctx.json?.ip_address != '' on_failure: - append: + tag: append_error_message_614c3056 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -182,6 +186,7 @@ processors: if: ctx.json?.record_created_at != null && ctx.json.record_created_at != '' on_failure: - append: + tag: append_error_message_b668c849 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -198,6 +203,7 @@ processors: if: ctx.json?.time != null && ctx.json.time != '' on_failure: - append: + tag: append_error_message_cadec2e8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -224,7 +230,7 @@ processors: if: ctx.user?.name != null - append: field: related.user - tag: append_audit_username_into_related_user + tag: append_related_user_fc96f0f9 value: '{{{user.email}}}' allow_duplicates: false if: ctx.user?.email != null @@ -274,6 +280,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/nozomi_networks/data_stream/health/elasticsearch/ingest_pipeline/default.yml b/packages/nozomi_networks/data_stream/health/elasticsearch/ingest_pipeline/default.yml index e1912dd1059..7c8fa135857 100644 --- a/packages/nozomi_networks/data_stream/health/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nozomi_networks/data_stream/health/elasticsearch/ingest_pipeline/default.yml @@ -39,6 +39,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -75,7 +76,7 @@ processors: ignore_empty_value: true - set: field: host.name - tag: set_host_hostname_from_health_appliance_host + tag: set_host_name_c1844dfc copy_from: nozomi_networks.health.appliance_host ignore_empty_value: true - append: @@ -109,6 +110,7 @@ processors: if: ctx.json?.appliance_ip != '' on_failure: - append: + tag: append_error_message_31f7cbc5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -147,6 +149,7 @@ processors: if: ctx.json?.record_created_at != null && ctx.json.record_created_at != '' on_failure: - append: + tag: append_error_message_a59536ce field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -167,6 +170,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_eff5d58d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -214,6 +218,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c0388d50 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -225,6 +230,7 @@ processors: if: ctx.json?.time != null && ctx.json.time != '' on_failure: - append: + tag: append_error_message_a0512bc1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: @@ -275,6 +281,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/nozomi_networks/data_stream/node/elasticsearch/ingest_pipeline/default.yml b/packages/nozomi_networks/data_stream/node/elasticsearch/ingest_pipeline/default.yml index 0fb788c1f71..059632e6246 100644 --- a/packages/nozomi_networks/data_stream/node/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nozomi_networks/data_stream/node/elasticsearch/ingest_pipeline/default.yml @@ -39,9 +39,11 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - dot_expander: + tag: dot_expander_*_6cdde46d field: "*" path: json - append: @@ -108,6 +110,7 @@ processors: if: ctx.json?.created_at != null on_failure: - append: + tag: append_error_message_d7f1b993 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -159,6 +162,7 @@ processors: if: ctx.json?.end_of_sale_date != null on_failure: - append: + tag: append_error_message_ee567b43 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -175,6 +179,7 @@ processors: if: ctx.json?.end_of_support_date != null on_failure: - append: + tag: append_error_message_adfa3437 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -201,6 +206,7 @@ processors: if: ctx.json?.first_activity_time != null on_failure: - append: + tag: append_error_message_9f47bd0f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -227,6 +233,7 @@ processors: if: ctx.json?.ip != '' on_failure: - append: + tag: append_error_message_cc9e6593 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -235,10 +242,12 @@ processors: copy_from: nozomi_networks.node.ip ignore_empty_value: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -270,6 +279,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8c0b7fbc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -280,6 +290,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_87bd3fca field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -290,6 +301,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_72cc354c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -300,6 +312,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_fa77f2e7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -310,6 +323,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_331db07e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -320,6 +334,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_1e23eaa3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -330,6 +345,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9f26f801 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -340,6 +356,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_81d22460 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -350,6 +367,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5ca62aee field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -360,6 +378,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ee9773ca field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -370,6 +389,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_1a636b45 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -380,6 +400,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_017c48e5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -401,6 +422,7 @@ processors: if: ctx.json?.last_activity_time != null on_failure: - append: + tag: append_error_message_2c4e8b3b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -428,6 +450,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_b84dfd09 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -453,6 +476,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_fc6de26d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -474,6 +498,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_aed5673a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - uppercase: @@ -482,6 +507,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4c748822 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -596,6 +622,7 @@ processors: target_field: nozomi_networks.node.properties.vendor.passive ignore_missing: true - foreach: + tag: foreach_json_protocols_8ec97444 field: json.protocols if: ctx.json?.protocols instanceof List processor: @@ -617,6 +644,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8d099267 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -632,6 +660,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_6d38eaec field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -642,6 +671,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_965fc860 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -652,6 +682,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_43e99a64 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -662,6 +693,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_6687e5d5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -672,6 +704,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_6914c11d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -682,6 +715,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2b658d47 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -692,6 +726,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_be493279 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -708,6 +743,7 @@ processors: if: ctx.json?.record_created_at != null on_failure: - append: + tag: append_error_message_f132d3cb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -721,6 +757,7 @@ processors: target_field: nozomi_networks.node.reputation ignore_missing: true - foreach: + tag: foreach_json_roles_005f44b1 field: json.roles if: ctx.json?.roles instanceof List processor: @@ -742,6 +779,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a90c730a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -757,6 +795,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_182e9459 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -767,6 +806,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_68e131d1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -777,6 +817,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_dc20b845 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -787,6 +828,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_0204958c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -797,6 +839,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_6c55d538 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -807,6 +850,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_195dbb7a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -817,6 +861,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5acbe9a0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -847,6 +892,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_70868625 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -857,6 +903,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_b5721366 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -867,6 +914,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_13c4386b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -877,6 +925,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_7c042765 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -887,6 +936,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a4134bdb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -897,6 +947,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_0b547d65 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -922,6 +973,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_aa2a4b8d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1013,6 +1065,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/nozomi_networks/data_stream/node_cve/elasticsearch/ingest_pipeline/default.yml b/packages/nozomi_networks/data_stream/node_cve/elasticsearch/ingest_pipeline/default.yml index 7f4e90dab59..7ec87cf52d2 100644 --- a/packages/nozomi_networks/data_stream/node_cve/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nozomi_networks/data_stream/node_cve/elasticsearch/ingest_pipeline/default.yml @@ -39,6 +39,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -99,6 +100,7 @@ processors: if: ctx.json?.appliance_ip != '' on_failure: - append: + tag: append_error_message_16856a80 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -164,6 +166,7 @@ processors: if: ctx.json?.cve_creation_time != null on_failure: - append: + tag: append_error_message_ddb21cda field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -184,6 +187,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_fc5f4766 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -199,6 +203,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_b015c1c8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -235,6 +240,7 @@ processors: if: ctx.json?.cve_update_time != null on_failure: - append: + tag: append_error_message_852435e2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -286,6 +292,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2b1ee6da field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -301,6 +308,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f1824bcb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -379,7 +387,7 @@ processors: ignore_missing: true - foreach: field: json.nodes - tag: foreach_nodes + tag: foreach_json_nodes_98ad5bbe if: ctx.json?.nodes instanceof List processor: append: @@ -422,6 +430,7 @@ processors: if: ctx.json?.record_created_at != null on_failure: - append: + tag: append_error_message_2f4057a2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -467,6 +476,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ac02d355 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -483,6 +493,7 @@ processors: if: ctx.json?.time != null on_failure: - append: + tag: append_error_message_21155f8e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -570,6 +581,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/nozomi_networks/data_stream/session/elasticsearch/ingest_pipeline/default.yml b/packages/nozomi_networks/data_stream/session/elasticsearch/ingest_pipeline/default.yml index 82b37ed28d6..7907c49322b 100644 --- a/packages/nozomi_networks/data_stream/session/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nozomi_networks/data_stream/session/elasticsearch/ingest_pipeline/default.yml @@ -39,6 +39,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -74,6 +75,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_491a95b2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -85,6 +87,7 @@ processors: if: ctx.json?.first_activity_time != null && ctx.json.first_activity_time != '' on_failure: - append: + tag: append_error_message_61d7c9bc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -104,7 +107,7 @@ processors: field: json.from tag: rename_from target_field: nozomi_networks.session.from - ignore_missing: true + ignore_missing: true - set: field: source.ip tag: set_source_ip_from_session_from_ip @@ -148,6 +151,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_95d318fc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -178,6 +182,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ff33d54c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -188,6 +193,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_53f57b49 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -198,6 +204,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c9e57efc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -214,6 +221,7 @@ processors: if: ctx.json?.last_activity_time != null && ctx.json.last_activity_time != '' on_failure: - append: + tag: append_error_message_51f9f088 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -244,6 +252,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2e3debf6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -258,7 +267,7 @@ processors: field: json.to tag: rename_to target_field: nozomi_networks.session.to - ignore_missing: true + ignore_missing: true - set: field: destination.ip tag: set_destination_ip_from_session_to_ip @@ -302,6 +311,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_56003859 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -327,6 +337,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f8d7c2a5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -337,6 +348,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_581bd59b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -347,6 +359,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5cac166e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -362,6 +375,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a31366e7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -372,6 +386,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_59df363c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -382,6 +397,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d891896e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -392,6 +408,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_bef28c40 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -407,6 +424,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_23f08fbd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -485,6 +503,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/nozomi_networks/data_stream/variable/elasticsearch/ingest_pipeline/default.yml b/packages/nozomi_networks/data_stream/variable/elasticsearch/ingest_pipeline/default.yml index 354cf2b1c92..931471e85d8 100644 --- a/packages/nozomi_networks/data_stream/variable/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nozomi_networks/data_stream/variable/elasticsearch/ingest_pipeline/default.yml @@ -39,9 +39,11 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - dot_expander: + tag: dot_expander_*_6cdde46d field: "*" path: json - set: @@ -85,6 +87,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4f547c71 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -96,6 +99,7 @@ processors: if: ctx.json?.first_activity_time != null on_failure: - append: + tag: append_error_message_ec4d3fa7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -116,6 +120,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_6947b9e2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -126,6 +131,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_fc26db2e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -136,6 +142,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4e52b8a5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -146,6 +153,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c7c7db78 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -161,6 +169,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2521cab1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -228,6 +237,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_63b1adf8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -244,6 +254,7 @@ processors: if: ctx.json?.last_activity_time != null on_failure: - append: + tag: append_error_message_0f83a2fb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -281,10 +292,12 @@ processors: copy_from: nozomi_networks.variable.last_client_ip ignore_empty_value: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -327,6 +340,7 @@ processors: if: ctx.json?.last_range_change_time != null on_failure: - append: + tag: append_error_message_fc9252d5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -338,6 +352,7 @@ processors: if: ctx.json?.last_update_time != null on_failure: - append: + tag: append_error_message_2b15f5bb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -349,6 +364,7 @@ processors: if: ctx.json?.last_valid_quality_time != null on_failure: - append: + tag: append_error_message_db48a421 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -359,6 +375,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_7cbfc84c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -369,6 +386,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e2a6b838 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -420,6 +438,7 @@ processors: if: ctx.json?.record_created_at != null on_failure: - append: + tag: append_error_message_98be4593 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -435,6 +454,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_0c5a5209 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -461,6 +481,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_b0c60597 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -514,6 +535,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/nozomi_networks/manifest.yml b/packages/nozomi_networks/manifest.yml index 2e598f654a4..74e482448b2 100644 --- a/packages/nozomi_networks/manifest.yml +++ b/packages/nozomi_networks/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: nozomi_networks title: Nozomi Networks -version: 0.3.1 +version: 0.3.2 description: Collect logs from Nozomi Networks with Elastic Agent. type: integration categories: diff --git a/packages/qualys_gav/changelog.yml b/packages/qualys_gav/changelog.yml index 0decfa183d3..3dcb8527445 100644 --- a/packages/qualys_gav/changelog.yml +++ b/packages/qualys_gav/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.9.3" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "0.9.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/qualys_gav/data_stream/asset/elasticsearch/ingest_pipeline/default.yml b/packages/qualys_gav/data_stream/asset/elasticsearch/ingest_pipeline/default.yml index 5e36ebec055..9422422510f 100644 --- a/packages/qualys_gav/data_stream/asset/elasticsearch/ingest_pipeline/default.yml +++ b/packages/qualys_gav/data_stream/asset/elasticsearch/ingest_pipeline/default.yml @@ -19,6 +19,7 @@ processors: if: ctx.interval_start != null && ctx.interval_start != '' on_failure: - append: + tag: append_error_message_3cdc5092 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: @@ -43,25 +44,29 @@ processors: description: >- Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - # Set up tag logic for choosing which cloud metadata to include. - append: + tag: append_tags_ae765f40 field: tags value: ["elastic_cloud_data", "provider_cloud_data"] if: ctx.tags?.contains('both') == true - script: + tag: script_023c6838 lang: painless if: ctx.tags?.contains('both') == true source: ctx.tags.remove(ctx.tags.indexOf('both')); - script: + tag: script_cfa1c445 lang: painless if: ctx.cloud == null && ctx.tags?.contains('elastic_cloud_data') == true source: ctx.tags.remove(ctx.tags.indexOf('elastic_cloud_data')); - set: + tag: set__conf_want_provider_cloud_4bad817b field: _conf.want_provider_cloud value: true if: ctx.tags?.contains('provider_cloud_data') == true - remove: + tag: remove_cloud_ddf0b5c6 description: Remove elastic agent-provided cloud metadata fields if not requested. field: cloud if: ctx.tags != null && !ctx.tags.contains('elastic_cloud_data') @@ -170,8 +175,10 @@ processors: if: ctx.qualys_gav?.asset?.address != '' on_failure: - remove: + tag: remove_qualys_gav_asset_address_0d5a702a field: qualys_gav.asset.address - append: + tag: append_error_message_549e6416 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -194,8 +201,10 @@ processors: if: ctx.qualys_gav?.asset?.agent?.connected_from != '' on_failure: - remove: + tag: remove_qualys_gav_asset_agent_connected_from_db5636ff field: qualys_gav.asset.agent.connected_from - append: + tag: append_error_message_9aa308a0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -214,8 +223,10 @@ processors: if: ctx.qualys_gav?.asset?.agent?.last_activity != null && ctx.qualys_gav.asset.agent.last_activity != '' on_failure: - remove: + tag: remove_qualys_gav_asset_agent_last_activity_eb3052f4 field: qualys_gav.asset.agent.last_activity - append: + tag: append_error_message_f3dd5d7b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -228,8 +239,10 @@ processors: if: ctx.qualys_gav?.asset?.agent?.last_checked_in != null && ctx.qualys_gav.asset.agent.last_checked_in != '' on_failure: - remove: + tag: remove_qualys_gav_asset_agent_last_checked_in_6c5fd0ec field: qualys_gav.asset.agent.last_checked_in - append: + tag: append_error_message_f7b4d077 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -242,8 +255,10 @@ processors: if: ctx.qualys_gav?.asset?.agent?.last_inventory != null && ctx.qualys_gav.asset.agent.last_inventory != '' on_failure: - remove: + tag: remove_qualys_gav_asset_agent_last_inventory_13c41b13 field: qualys_gav.asset.agent.last_inventory - append: + tag: append_error_message_6c5f5583 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -254,8 +269,10 @@ processors: if: ctx.qualys_gav?.asset?.agent?.udc_manifest_assigned != '' on_failure: - remove: + tag: remove_qualys_gav_asset_agent_udc_manifest_assigned_c69b65f0 field: qualys_gav.asset.agent.udc_manifest_assigned - append: + tag: append_error_message_cd44145c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -266,8 +283,10 @@ processors: if: ctx.qualys_gav?.asset?.agent?.error_status != '' on_failure: - remove: + tag: remove_qualys_gav_asset_agent_error_status_b1c7fde8 field: qualys_gav.asset.agent.error_status - append: + tag: append_error_message_3fc99ffa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' # Remove fields having 0 value @@ -288,8 +307,10 @@ processors: if: ctx.qualys_gav?.asset?.sensor?.last_vmscan != null && ctx.qualys_gav.asset.sensor.last_vmscan != '' on_failure: - remove: + tag: remove_qualys_gav_asset_sensor_last_vmscan_808540e4 field: qualys_gav.asset.sensor.last_vmscan - append: + tag: append_error_message_cfb15aeb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -302,8 +323,10 @@ processors: if: ctx.qualys_gav?.asset?.sensor?.last_compliance_scan != null && ctx.qualys_gav.asset.sensor.last_compliance_scan != '' on_failure: - remove: + tag: remove_qualys_gav_asset_sensor_last_compliance_scan_c14e34ae field: qualys_gav.asset.sensor.last_compliance_scan - append: + tag: append_error_message_15802628 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -316,8 +339,10 @@ processors: if: ctx.qualys_gav?.asset?.sensor?.last_full_scan != null && ctx.qualys_gav.asset.sensor.last_full_scan != '' on_failure: - remove: + tag: remove_qualys_gav_asset_sensor_last_full_scan_b7013a00 field: qualys_gav.asset.sensor.last_full_scan - append: + tag: append_error_message_6dc15e9c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -330,8 +355,10 @@ processors: if: ctx.qualys_gav?.asset?.sensor?.last_vm_scan_date_scanner != null && ctx.qualys_gav.asset.sensor.last_vm_scan_date_scanner != '' on_failure: - remove: + tag: remove_qualys_gav_asset_sensor_last_vm_scan_date_scanner_83b1afac field: qualys_gav.asset.sensor.last_vm_scan_date_scanner - append: + tag: append_error_message_4b7cdc22 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -344,8 +371,10 @@ processors: if: ctx.qualys_gav?.asset?.sensor?.first_easm_scan_date != null && ctx.qualys_gav.asset.sensor.first_easm_scan_date != '' on_failure: - remove: + tag: remove_qualys_gav_asset_sensor_first_easm_scan_date_38825704 field: qualys_gav.asset.sensor.first_easm_scan_date - append: + tag: append_error_message_575d7e2c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -358,8 +387,10 @@ processors: if: ctx.qualys_gav?.asset?.sensor?.last_vm_scan_date_agent != null && ctx.qualys_gav.asset.sensor.last_vm_scan_date_agent != '' on_failure: - remove: + tag: remove_qualys_gav_asset_sensor_last_vm_scan_date_agent_cfcc4641 field: qualys_gav.asset.sensor.last_vm_scan_date_agent - append: + tag: append_error_message_df5d0892 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -372,8 +403,10 @@ processors: if: ctx.qualys_gav?.asset?.sensor?.last_pc_scan_date_scanner != null && ctx.qualys_gav.asset.sensor.last_pc_scan_date_scanner != '' on_failure: - remove: + tag: remove_qualys_gav_asset_sensor_last_pc_scan_date_scanner_2c1f4aec field: qualys_gav.asset.sensor.last_pc_scan_date_scanner - append: + tag: append_error_message_43d142be field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -386,8 +419,10 @@ processors: if: ctx.qualys_gav?.asset?.sensor?.last_pc_scan_date_agent != null && ctx.qualys_gav.asset.sensor.last_pc_scan_date_agent != '' on_failure: - remove: + tag: remove_qualys_gav_asset_sensor_last_pc_scan_date_agent_5153e27d field: qualys_gav.asset.sensor.last_pc_scan_date_agent - append: + tag: append_error_message_01013632 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -400,8 +435,10 @@ processors: if: ctx.qualys_gav?.asset?.sensor?.last_easm_scan_date != null && ctx.qualys_gav.asset.sensor.last_easm_scan_date != '' on_failure: - remove: + tag: remove_qualys_gav_asset_sensor_last_easm_scan_date_800b1ff2 field: qualys_gav.asset.sensor.last_easm_scan_date - append: + tag: append_error_message_870aea82 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -459,8 +496,10 @@ processors: if: ctx.qualys_gav?.asset?.container?.no_of_containers != '' on_failure: - remove: + tag: remove_qualys_gav_asset_container_no_of_containers_d939893f field: qualys_gav.asset.container.no_of_containers - append: + tag: append_error_message_806d532c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -471,8 +510,10 @@ processors: if: ctx.qualys_gav?.asset?.container?.no_of_images != '' on_failure: - remove: + tag: remove_qualys_gav_asset_container_no_of_images_063a2749 field: qualys_gav.asset.container.no_of_images - append: + tag: append_error_message_2e6f2388 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -483,8 +524,10 @@ processors: if: ctx.qualys_gav?.asset?.cpu_count != '' on_failure: - remove: + tag: remove_qualys_gav_asset_cpu_count_aaa21c40 field: qualys_gav.asset.cpu_count - append: + tag: append_error_message_c214ca06 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -497,8 +540,10 @@ processors: if: ctx.qualys_gav?.asset?.created_date != null && ctx.qualys_gav.asset.created_date != '' on_failure: - remove: + tag: remove_qualys_gav_asset_created_date_705df3ae field: qualys_gav.asset.created_date - append: + tag: append_error_message_95b697b7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -514,8 +559,10 @@ processors: if: ctx.qualys_gav?.asset?.criticality?.is_default != '' on_failure: - remove: + tag: remove_qualys_gav_asset_criticality_is_default_636ba50b field: qualys_gav.asset.criticality.is_default - append: + tag: append_error_message_2e07343c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -528,8 +575,10 @@ processors: if: ctx.qualys_gav?.asset?.criticality?.last_updated != null && ctx.qualys_gav.asset.criticality.last_updated != '' on_failure: - remove: + tag: remove_qualys_gav_asset_criticality_last_updated_181d4b3e field: qualys_gav.asset.criticality.last_updated - append: + tag: append_error_message_47c3d483 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -540,8 +589,10 @@ processors: if: ctx.qualys_gav?.asset?.criticality?.score != '' on_failure: - remove: + tag: remove_qualys_gav_asset_criticality_score_f0c6a845 field: qualys_gav.asset.criticality.score - append: + tag: append_error_message_a4fb47ea field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -595,8 +646,10 @@ processors: if: ctx.qualys_gav?.asset?.hardware?.lifecycle?.eos_date != null && ctx.qualys_gav.asset.hardware.lifecycle.eos_date != '' on_failure: - remove: + tag: remove_qualys_gav_asset_hardware_lifecycle_eos_date_5b63d3fd field: qualys_gav.asset.hardware.lifecycle.eos_date - append: + tag: append_error_message_bc086db7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -609,8 +662,10 @@ processors: if: ctx.qualys_gav?.asset?.hardware?.lifecycle?.ga_date != null && ctx.qualys_gav.asset.hardware.lifecycle.ga_date != '' on_failure: - remove: + tag: remove_qualys_gav_asset_hardware_lifecycle_ga_date_c3032b5e field: qualys_gav.asset.hardware.lifecycle.ga_date - append: + tag: append_error_message_26629491 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -623,8 +678,10 @@ processors: if: ctx.qualys_gav?.asset?.hardware?.lifecycle?.intro_date != null && ctx.qualys_gav.asset.hardware.lifecycle.intro_date != '' on_failure: - remove: + tag: remove_qualys_gav_asset_hardware_lifecycle_intro_date_f47bf89c field: qualys_gav.asset.hardware.lifecycle.intro_date - append: + tag: append_error_message_4e6e69df field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -637,8 +694,10 @@ processors: if: ctx.qualys_gav?.asset?.hardware?.lifecycle?.obsolete_date != null && ctx.qualys_gav.asset.hardware.lifecycle.obsolete_date != '' on_failure: - remove: + tag: remove_qualys_gav_asset_hardware_lifecycle_obsolete_date_4c168eb9 field: qualys_gav.asset.hardware.lifecycle.obsolete_date - append: + tag: append_error_message_3d56f317 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -677,8 +736,10 @@ processors: if: ctx.qualys_gav?.asset?.inventory?.created != null && ctx.qualys_gav.asset.inventory.created != '' on_failure: - remove: + tag: remove_qualys_gav_asset_inventory_created_3621ec17 field: qualys_gav.asset.inventory.created - append: + tag: append_error_message_c79758e5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -691,8 +752,10 @@ processors: if: ctx.qualys_gav?.asset?.inventory?.last_updated != null && ctx.qualys_gav.asset.inventory.last_updated != '' on_failure: - remove: + tag: remove_qualys_gav_asset_inventory_last_updated_76692d3f field: qualys_gav.asset.inventory.last_updated - append: + tag: append_error_message_8b25398b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -737,8 +800,10 @@ processors: if: ctx.qualys_gav?.asset?.activity?.last_scanned_date != null && ctx.qualys_gav.asset.activity.last_scanned_date != '' on_failure: - remove: + tag: remove_qualys_gav_asset_activity_last_scanned_date_83ecfaee field: qualys_gav.asset.activity.last_scanned_date - append: + tag: append_error_message_cf33fa95 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -749,8 +814,10 @@ processors: if: ctx.qualys_gav?.asset?.is_container_host != '' on_failure: - remove: + tag: remove_qualys_gav_asset_is_container_host_df8f9173 field: qualys_gav.asset.is_container_host - append: + tag: append_error_message_0472b0ba field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -763,8 +830,10 @@ processors: if: ctx.qualys_gav?.asset?.last_boot != null && ctx.qualys_gav.asset.last_boot != '' on_failure: - remove: + tag: remove_qualys_gav_asset_last_boot_fb1aa69a field: qualys_gav.asset.last_boot - append: + tag: append_error_message_295822fd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1047,12 +1116,12 @@ processors: lang: painless if: ctx.qualys_gav?.asset?.operating_system?.category1 != null params: - macos : macos - linux : linux - unix : unix - windows : windows - ios : ios - android : android + macos: macos + linux: linux + unix: unix + windows: windows + ios: ios + android: android source: | def os_type = ctx.qualys_gav.asset.operating_system.category1.toLowerCase(); @@ -1084,8 +1153,10 @@ processors: if: ctx.qualys_gav?.asset?.operating_system?.install_date != null && ctx.qualys_gav.asset.operating_system.install_date != '' on_failure: - remove: + tag: remove_qualys_gav_asset_operating_system_install_date_dd332c8a field: qualys_gav.asset.operating_system.install_date - append: + tag: append_error_message_e86e0da1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -1098,8 +1169,10 @@ processors: if: ctx.qualys_gav?.asset?.operating_system?.lifecycle?.eol_date != null && ctx.qualys_gav.asset.operating_system.lifecycle.eol_date != '' on_failure: - remove: + tag: remove_qualys_gav_asset_operating_system_lifecycle_eol_date_5ea340ed field: qualys_gav.asset.operating_system.lifecycle.eol_date - append: + tag: append_error_message_b203b637 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -1112,8 +1185,10 @@ processors: if: ctx.qualys_gav?.asset?.operating_system?.lifecycle?.eos_date != null && ctx.qualys_gav.asset.operating_system.lifecycle.eos_date != '' on_failure: - remove: + tag: remove_qualys_gav_asset_operating_system_lifecycle_eos_date_29ce926e field: qualys_gav.asset.operating_system.lifecycle.eos_date - append: + tag: append_error_message_8b8c416f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -1126,8 +1201,10 @@ processors: if: ctx.qualys_gav?.asset?.operating_system?.lifecycle?.ga_date != null && ctx.qualys_gav.asset.operating_system.lifecycle.ga_date != '' on_failure: - remove: + tag: remove_qualys_gav_asset_operating_system_lifecycle_ga_date_6efa9a05 field: qualys_gav.asset.operating_system.lifecycle.ga_date - append: + tag: append_error_message_1ac03533 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1138,8 +1215,10 @@ processors: if: ctx.qualys_gav?.asset?.operating_system?.lifecycle?.detection_score != '' on_failure: - remove: + tag: remove_qualys_gav_asset_operating_system_lifecycle_detection_score_4b9ccf3d field: qualys_gav.asset.operating_system.lifecycle.detection_score - append: + tag: append_error_message_86dace88 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1181,9 +1260,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_qualys_gav_asset_cloud_provider_aws_ec2_has_agent_0cdce69b field: qualys_gav.asset.cloud_provider.aws.ec2.has_agent ignore_missing: true - append: + tag: append_error_message_63a1c0be field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -1196,9 +1277,11 @@ processors: if: ctx.qualys_gav?.asset?.cloud_provider?.aws?.ec2?.launchdate != null && ctx.qualys_gav.asset.cloud_provider.aws.ec2.launchdate != '' on_failure: - remove: + tag: remove_qualys_gav_asset_cloud_provider_aws_ec2_launchdate_8e78ea70 field: qualys_gav.asset.cloud_provider.aws.ec2.launchdate ignore_missing: true - append: + tag: append_error_message_3bfa46e8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1208,9 +1291,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_qualys_gav_asset_cloud_provider_aws_ec2_qualys_scanner_f49466f1 field: qualys_gav.asset.cloud_provider.aws.ec2.qualys_scanner ignore_missing: true - append: + tag: append_error_message_5ae5e698 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1220,9 +1305,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_qualys_gav_asset_cloud_provider_aws_ec2_spot_instance_cdbd0f4b field: qualys_gav.asset.cloud_provider.aws.ec2.spot_instance ignore_missing: true - append: + tag: append_error_message_ecb043c2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' @@ -1302,7 +1389,7 @@ processors: ignore_empty_value: true - set: field: cloud.service.name - tag: set_cloud_service_name + tag: set_cloud_service_name_6f4b3b6c if: ctx._conf?.want_provider_cloud == true && ctx.cloud?.provider == 'azure' && ctx.qualys_gav?.asset?.cloud_provider?.azure?.vm != null value: vm @@ -1352,11 +1439,12 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e21b85a8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: cloud.service.name - tag: set_cloud_service_name + tag: set_cloud_service_name_dcdaf8c0 if: ctx._conf?.want_provider_cloud == true && ctx.cloud?.provider == 'gcp' && ctx.qualys_gav?.asset?.cloud_provider?.gcp?.compute != null value: compute @@ -1371,7 +1459,7 @@ processors: tag: convert_qualys_gav_asset_cloud_provider_ibm_virtual_server_ibm_id_to_string type: string ignore_missing: true - + # Map/Convert IBM specific fields to ECS fields - set: field: cloud.account.id @@ -1399,7 +1487,7 @@ processors: ignore_empty_value: true - set: field: cloud.service.name - tag: set_cloud_service_name + tag: set_cloud_service_name_93672477 if: ctx._conf?.want_provider_cloud == true && ctx.cloud?.provider == 'azure' && ctx.qualys_gav?.asset?.cloud_provider?.ibm?.virtual_server != null value: virtual_server @@ -1412,8 +1500,10 @@ processors: if: ctx.qualys_gav?.asset?.risk_score != '' on_failure: - remove: + tag: remove_qualys_gav_asset_risk_score_46898134 field: qualys_gav.asset.risk_score - append: + tag: append_error_message_4bc086e0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1441,8 +1531,10 @@ processors: if: ctx.qualys_gav?.asset?.sensor_last_updated_date != null && ctx.qualys_gav.asset.sensor_last_updated_date != '' on_failure: - remove: + tag: remove_qualys_gav_asset_sensor_last_updated_date_e7aa2ebb field: qualys_gav.asset.sensor_last_updated_date - append: + tag: append_error_message_18f0e285 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1824,8 +1916,10 @@ processors: if: ctx.qualys_gav?.asset?.threads_per_core != '' on_failure: - remove: + tag: remove_qualys_gav_asset_threads_per_core_2144e1fc field: qualys_gav.asset.threads_per_core - append: + tag: append_error_message_abc82d01 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1841,8 +1935,10 @@ processors: if: ctx.qualys_gav?.asset?.total_memory != '' on_failure: - remove: + tag: remove_qualys_gav_asset_total_memory_d78e6fd6 field: qualys_gav.asset.total_memory - append: + tag: append_error_message_730ab154 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1947,8 +2043,10 @@ processors: if: ctx.qualys_gav?.asset?.last_modified_date != null && ctx.qualys_gav.asset.last_modified_date != '' on_failure: - remove: + tag: remove_qualys_gav_asset_last_modified_date_ca6e9e50 field: qualys_gav.asset.last_modified_date - append: + tag: append_error_message_52a15937 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1959,8 +2057,10 @@ processors: if: ctx.qualys_gav?.asset?.processor?.speed != '' on_failure: - remove: + tag: remove_qualys_gav_asset_processor_speed_36a66ad5 field: qualys_gav.asset.processor.speed - append: + tag: append_error_message_c0a0bad4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1971,8 +2071,10 @@ processors: if: ctx.qualys_gav?.asset?.processor?.num_cpus != '' on_failure: - remove: + tag: remove_qualys_gav_asset_processor_num_cpus_ee19437d field: qualys_gav.asset.processor.num_cpus - append: + tag: append_error_message_281bd05f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1983,8 +2085,10 @@ processors: if: ctx.qualys_gav?.asset?.processor?.no_of_socket != '' on_failure: - remove: + tag: remove_qualys_gav_asset_processor_no_of_socket_d60370fa field: qualys_gav.asset.processor.no_of_socket - append: + tag: append_error_message_065b4e21 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1995,8 +2099,10 @@ processors: if: ctx.qualys_gav?.asset?.processor?.threads_per_core != '' on_failure: - remove: + tag: remove_qualys_gav_asset_processor_threads_per_core_b07b85e8 field: qualys_gav.asset.processor.threads_per_core - append: + tag: append_error_message_8ab88271 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2007,8 +2113,10 @@ processors: if: ctx.qualys_gav?.asset?.processor?.cores_per_socket != '' on_failure: - remove: + tag: remove_qualys_gav_asset_processor_cores_per_socket_13389faf field: qualys_gav.asset.processor.cores_per_socket - append: + tag: append_error_message_2c067477 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2115,6 +2223,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/qualys_gav/manifest.yml b/packages/qualys_gav/manifest.yml index dbd8411eb89..8bf42b37f18 100644 --- a/packages/qualys_gav/manifest.yml +++ b/packages/qualys_gav/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.4.0 name: qualys_gav title: Qualys Global AssetView -version: 0.9.2 +version: 0.9.3 description: Collect logs from Qualys Global AssetView with Elastic Agent. type: integration categories: diff --git a/packages/qualys_was/changelog.yml b/packages/qualys_was/changelog.yml index bf4cefb9298..c07b5c7bf7f 100644 --- a/packages/qualys_was/changelog.yml +++ b/packages/qualys_was/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.5.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "0.5.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/qualys_was/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml b/packages/qualys_was/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml index bbd06f08ee6..5853418aef0 100644 --- a/packages/qualys_was/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml +++ b/packages/qualys_was/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml @@ -5,31 +5,37 @@ processors: tag: data_collection_error if: ctx.error?.message != null && ctx.message == null && ctx.event?.original == null - rename: + tag: rename_message_to_event_original_c35a9364 field: message target_field: event.original if: ctx.event?.original == null description: 'Renames the original `message` field to `event.original` to store a copy of the original message.' on_failure: - append: + tag: append_error_message_9219f8a2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - set: + tag: set_event_created_3b69f632 if: ctx['@timestamp'] != null field: event.created copy_from: '@timestamp' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_ecs_version_135371bc field: ecs.version value: '8.16.0' - terminate: - tag: data_collection_error + tag: terminate_3c149bdd if: ctx.error?.message != null && ctx.message == null && ctx.event?.original == null description: error message set and no data to process. - set: @@ -37,28 +43,36 @@ processors: tag: set_event_kind_1 value: event - set: + tag: set_event_category_13b30ef2 field: event.category value: [vulnerability] - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] - set: + tag: set_vulnerability_classification_8b34b171 field: vulnerability.classification value: CVSS - set: + tag: set_vulnerability_enumeration_c3d6bae0 field: vulnerability.enumeration value: CWE - set: + tag: set_vulnerability_category_8b0e7a6a field: vulnerability.category value: [Web Application] - set: + tag: set_vulnerability_scanner_vendor_54e04563 field: vulnerability.scanner.vendor value: Qualys - json: + tag: json_event_original_to_json_2ab742de field: event.original target_field: json on_failure: - append: + tag: append_error_message_c9525988 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -94,11 +108,12 @@ processors: } on_failure: - append: + tag: append_error_message_df4a45fa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: if: ctx.qualys_was?.vulnerability?.qid != null - tag: vul_id_is_long + tag: script_d94403e4 lang: painless source: > if (ctx.qualys_was.vulnerability.qid instanceof String) { @@ -108,6 +123,7 @@ processors: } on_failure: - append: + tag: append_error_message_24297e4e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -124,6 +140,7 @@ processors: if: ctx.json?.detection?.lastDetectedDate != null && ctx.json.detection.lastDetectedDate != '' on_failure: - append: + tag: append_error_message_d75914ef field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -135,6 +152,7 @@ processors: if: ctx.json?.detection?.firstDetectedDate != null && ctx.json.detection.firstDetectedDate != '' on_failure: - append: + tag: append_error_message_5e842a89 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -145,6 +163,7 @@ processors: target_field: qualys_was.vulnerability.last_test_datetime on_failure: - append: + tag: append_error_message_e779d17f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -156,6 +175,7 @@ processors: if: ctx.json?.detection?.updatedDate != null && ctx.json.detection.updatedDate != '' on_failure: - append: + tag: append_error_message_97a7e165 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -167,6 +187,7 @@ processors: if: ctx.json?.detection?.fixedDate != null && ctx.json.detection.fixedDate != '' on_failure: - append: + tag: append_error_message_c140f81e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -181,14 +202,15 @@ processors: - rename: field: json.detection.potential tag: rename_potential - target_field: qualys_was.vulnerability.potential + target_field: qualys_was.vulnerability.potential ignore_missing: true - rename: field: json.detection.param tag: rename_param - target_field: qualys_was.vulnerability.param + target_field: qualys_was.vulnerability.param ignore_missing: true - script: + tag: script_0c1121dd if: ctx.json?.detection?.detectionScore != null lang: painless source: > @@ -199,9 +221,11 @@ processors: } on_failure: - append: + tag: append_error_message_a62cb073 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: + tag: script_9af9e5ad if: ctx.json?.detection?.timesDetected != null lang: painless source: > @@ -212,24 +236,26 @@ processors: } on_failure: - append: + tag: append_error_message_00b5fe93 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: field: json.detection.webApp.name tag: rename_webApp_name - target_field: qualys_was.vulnerability.web_app.name + target_field: qualys_was.vulnerability.web_app.name ignore_missing: true - rename: field: json.detection.webApp.url tag: rename_webApp_url - target_field: qualys_was.vulnerability.web_app.url + target_field: qualys_was.vulnerability.web_app.url ignore_missing: true - rename: field: json.detection.webApp.id - tag: rename_webApp_url - target_field: qualys_was.vulnerability.web_app.id + tag: rename_json_detection_webApp_id_to_qualys_was_vulnerability_web_app_id_de71119f + target_field: qualys_was.vulnerability.web_app.id ignore_missing: true - script: + tag: script_9ac6595e if: ctx.qualys_was?.vulnerability?.web_app?.id != null lang: painless source: > @@ -240,6 +266,7 @@ processors: } on_failure: - append: + tag: append_error_message_34fe91bc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -262,12 +289,13 @@ processors: ctx.qualys_was.vulnerability.web_app.tags = tagsList; on_failure: - append: + tag: append_error_message_b0076e78 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: field: json.detection.type tag: rename_type - target_field: qualys_was.vulnerability.type + target_field: qualys_was.vulnerability.type - rename: field: json.detection.status tag: rename_status @@ -298,6 +326,7 @@ processors: target_field: qualys_was.vulnerability.ignoredBy.id ignore_missing: true - script: + tag: script_78116e44 if: ctx.qualys_was?.vulnerability?.ignoredBy?.id != null && ctx.qualys_was.vulnerability.ignoredBy.id != "" lang: painless source: > @@ -308,6 +337,7 @@ processors: } on_failure: - append: + tag: append_error_message_62d80e16 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -335,6 +365,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8e3259c9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' @@ -365,6 +396,7 @@ processors: } on_failure: - append: + tag: append_error_message_1ff4112d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -387,9 +419,11 @@ processors: ctx.qualys_was.vulnerability.wasc_references = wascList; on_failure: - append: + tag: append_error_message_a0fee976 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: + tag: script_65779699 description: 'Create list of owasp references' if: ctx.json?.detection?.owasp?.list != null && ctx.json?.detection?.owasp?.list.size() > 0 lang: painless @@ -408,6 +442,7 @@ processors: ctx.qualys_was.vulnerability.owasp_references = owaspList; on_failure: - append: + tag: append_error_message_0ba1f82b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: @@ -417,6 +452,7 @@ processors: # Handle knowledge_base - append: + tag: append_error_message_22b416f3 field: error.message if: ctx.json?.Finding.knowledge_base == null || ctx.json.Finding.knowledge_base.size() == 0 value: 'Qualys QID not found in the Knowledge Base' @@ -426,6 +462,7 @@ processors: tag: pipeline_knowledge_base ignore_missing_pipeline: true - rename: + tag: rename_message_to_event_original_35631c5d field: message target_field: event.original ignore_missing: true @@ -433,7 +470,7 @@ processors: - pipeline: name: '{{ IngestPipeline "pipeline_knowledge_base" }}' if: ctx.Finding != null && ctx.Finding.knowledge_base != null && ctx.Finding.knowledge_base.size() > 0 - tag: pipeline_knowledge_base + tag: pipeline_693c5eae ignore_missing_pipeline: true - append: field: event.kind @@ -443,6 +480,7 @@ processors: if: ctx.error?.message != null allow_duplicates: false - append: + tag: append_tags_0f0a9140 field: tags value: preserve_original_event allow_duplicates: false @@ -451,8 +489,8 @@ processors: on_failure: - append: field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} - in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: field: event.kind value: pipeline_error diff --git a/packages/qualys_was/data_stream/vulnerability/elasticsearch/ingest_pipeline/pipeline_knowledge_base.yml b/packages/qualys_was/data_stream/vulnerability/elasticsearch/ingest_pipeline/pipeline_knowledge_base.yml index 72ea73559b2..5b6a1adc614 100644 --- a/packages/qualys_was/data_stream/vulnerability/elasticsearch/ingest_pipeline/pipeline_knowledge_base.yml +++ b/packages/qualys_was/data_stream/vulnerability/elasticsearch/ingest_pipeline/pipeline_knowledge_base.yml @@ -37,7 +37,7 @@ processors: tag: rename_COMPLIANCE_LIST_COMPLIANCE_DESCRIPTION target_field: qualys_was.vulnerability.knowledge_base.compliance_list.description ignore_missing: true -# Handle case when COMPLIANCE_LIST.COMPLIANCE is an array + # Handle case when COMPLIANCE_LIST.COMPLIANCE is an array - foreach: field: json.knowledge_base.COMPLIANCE_LIST.COMPLIANCE if: ctx.qualys_was?.vulnerability?.knowledge_base?.COMPLIANCE_LIST?.COMPLIANCE instanceof List @@ -107,7 +107,7 @@ processors: ignore_missing: true - rename: field: json.knowledge_base.QID - tag: rename_QID + tag: rename_json_knowledge_base_QID_to_qualys_was_vulnerability_knowledge_base_qid_35560f2c target_field: qualys_was.vulnerability.knowledge_base.qid ignore_missing: true - rename: @@ -130,7 +130,7 @@ processors: tag: rename_TITLE target_field: qualys_was.vulnerability.knowledge_base.title ignore_missing: true -# Handle case when BUGTRAQ_LIST.BUGTRAQ is an object + # Handle case when BUGTRAQ_LIST.BUGTRAQ is an object - rename: field: json.knowledge_base.BUGTRAQ_LIST.BUGTRAQ.ID tag: rename_BUGTRAQ_LIST_BUGTRAQ_ID @@ -141,7 +141,7 @@ processors: tag: rename_BUGTRAQ_LIST_BUGTRAQ_URL target_field: qualys_was.vulnerability.knowledge_base.bugtraq_list.url ignore_missing: true -# Handle case when BUGTRAQ_LIST.BUGTRAQ is an array + # Handle case when BUGTRAQ_LIST.BUGTRAQ is an array - foreach: field: json.knowledge_base.BUGTRAQ_LIST.BUGTRAQ if: ctx.json?.knowledge_base?.BUGTRAQ_LIST?.BUGTRAQ instanceof List @@ -186,7 +186,7 @@ processors: if: ctx.json?.knowledge_base?.CVSS?.BASE instanceof String - rename: field: json.knowledge_base.CVSS.BASE - tag: rename_CVSS_BASE + tag: rename_json_knowledge_base_CVSS_BASE_to_qualys_was_vulnerability_knowledge_base_cvss_base_obj_06e1b687 target_field: qualys_was.vulnerability.knowledge_base.cvss.base_obj ignore_missing: true if: ctx.json?.knowledge_base?.CVSS?.BASE instanceof Object @@ -347,6 +347,7 @@ processors: if: ctx.vulnerability?.reference != null && !(ctx.vulnerability.reference instanceof List) on_failure: - append: + tag: append_error_message_ccfa865f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' # Handle case when EXPLT_SRC and EXPLT_LIST.EXPLT is an object @@ -496,6 +497,7 @@ processors: if: ctx.json?.knowledge_base?.DISCOVERY?.REMOTE != '' on_failure: - append: + tag: append_error_message_d25f745d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' # Handle case when SOFTWARE_LIST.SOFTWARE is an object @@ -583,6 +585,7 @@ processors: if: ctx.json?.knowledge_base?.LAST_SERVICE_MODIFICATION_DATETIME != null && ctx.qualys_was.vulnerability.knowledge_base.LAST_SERVICE_MODIFICATION_DATETIME != '' on_failure: - append: + tag: append_error_message_8c705c64 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -594,6 +597,7 @@ processors: if: ctx.json?.knowledge_base?.LAST_CUSTOMIZATION?.DATETIME != null && ctx.qualys_was.vulnerability.knowledge_base.LAST_CUSTOMIZATION.DATETIME != '' on_failure: - append: + tag: append_error_message_c0ac2860 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -610,6 +614,7 @@ processors: if: ctx.json?.knowledge_base?.PUBLISHED_DATETIME != null && ctx.qualys_was.vulnerability.knowledge_base.PUBLISHED_DATETIME != '' on_failure: - append: + tag: append_error_message_0fd6a546 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' # Handle case when CHANGE_LOG_LIST.CHANGE_LOG_INFO is an object @@ -632,6 +637,7 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_9f1a2e73 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' # Handle case when CHANGE_LOG_LIST.CHANGE_LOG_INFO is an array @@ -696,6 +702,7 @@ processors: if: ctx.json?.knowledge_base?.IS_DISABLED != '' on_failure: - append: + tag: append_error_message_599d4103 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -707,7 +714,7 @@ processors: field: json.knowledge_base.PATCHABLE tag: set_PATCHABLE_false value: false - if: ctx.json?.knowledge_base?.PATCHABLE == '0' + if: ctx.json?.knowledge_base?.PATCHABLE == '0' - convert: field: json.knowledge_base.PATCHABLE tag: convert_PATCHABLE_to_boolean @@ -717,6 +724,7 @@ processors: if: ctx.json?.knowledge_base?.PATCHABLE != '' on_failure: - append: + tag: append_error_message_ab46bb45 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -738,6 +746,7 @@ processors: if: ctx.json?.knowledge_base?.PCI_FLAG != '' on_failure: - append: + tag: append_error_message_6253e86b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -760,6 +769,7 @@ processors: ctx.vulnerability.severity = params.vuln_level.getOrDefault(level, params.vuln_level["0"]); on_failure: - append: + tag: append_error_message_e8a952ee field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -791,6 +801,7 @@ processors: drop(ctx); on_failure: - append: + tag: append_error_message_2dc525b4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: diff --git a/packages/qualys_was/manifest.yml b/packages/qualys_was/manifest.yml index 83df78975e6..ee78c9b86c6 100644 --- a/packages/qualys_was/manifest.yml +++ b/packages/qualys_was/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.4.0" name: qualys_was title: Qualys Web Application Scanning (WAS) -version: "0.5.1" +version: "0.5.2" description: Collect data from Qualys Web Application Scanning platform with Elastic Agent or Agentless type: integration categories: diff --git a/packages/swimlane/changelog.yml b/packages/swimlane/changelog.yml index e7fcef5e670..5714a17fc9e 100644 --- a/packages/swimlane/changelog.yml +++ b/packages/swimlane/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.5.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "0.5.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/swimlane/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml b/packages/swimlane/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml index 296866c9b0d..cc97a4102ea 100644 --- a/packages/swimlane/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/swimlane/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml @@ -2,25 +2,31 @@ description: Pipeline for Swimlane Audit Logs processors: - set: + tag: set_event_original_b9758751 field: event.original copy_from: message - json: + tag: json_message_to_json_576c813b field: message target_field: json ignore_failure: true - drop: + tag: drop_631a5346 if: ctx.json == null || !(ctx.json instanceof Map) - remove: - field: + tag: remove_974dbe77 + field: - message - input - log ignore_missing: true - rename: + tag: rename_json_to_swimlane_audit_log_d2655548 field: json target_field: swimlane.audit_log ignore_missing: true - drop: + tag: drop_5063c354 if: >- ctx.swimlane?.audit_log?.description != null && ( @@ -28,35 +34,43 @@ processors: ctx.swimlane.audit_log.description.toLowerCase().contains('created a new record') ) - json: + tag: json_swimlane_audit_log_newValue_561e4375 field: swimlane.audit_log.newValue if: ctx.swimlane?.audit_log?.newValue != null ignore_failure: true - rename: + tag: rename_swimlane_audit_log_user_to_user_name_dbbc9be1 field: swimlane.audit_log.user target_field: user.name ignore_missing: true - append: + tag: append_related_user_837e080f field: related.user value: '{{{user.name}}}' allow_duplicates: false if: ctx.user?.name != null - rename: + tag: rename_swimlane_audit_log_accountId_to_cloud_origin_account_id_f309b8fd field: swimlane.audit_log.accountId target_field: cloud.origin.account.id ignore_missing: true - rename: + tag: rename_swimlane_audit_log_tenantId_to_cloud_origin_project_id_a841958a field: swimlane.audit_log.tenantId target_field: cloud.origin.project.id ignore_missing: true - rename: + tag: rename_swimlane_audit_log_logLevel_to_log_level_519425d8 field: swimlane.audit_log.logLevel target_field: log.level ignore_missing: true - rename: + tag: rename_swimlane_audit_log_path_to_url_path_0515a08d field: swimlane.audit_log.path target_field: url.path ignore_missing: true - script: + tag: script_0fb4f610 lang: painless description: Normalize SourceIp to a single value source: >- @@ -69,45 +83,55 @@ processors: } } - grok: + tag: grok_swimlane_audit_log_sourceIp_30e91ffc field: swimlane.audit_log.sourceIp patterns: - '::ffff:%{IPV4:source.ip}' - '%{IPV4:source.ip}' - - '%{IPV6:source.ip}' + - '%{IPV6:source.ip}' ignore_failure: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - rename: + tag: rename_swimlane_audit_log_tenant_to_cloud_origin_tenant_id_4e46bff8 field: swimlane.audit_log.tenant target_field: cloud.origin.tenant.id ignore_missing: true - rename: + tag: rename_swimlane_audit_log_category_to_log_category_fa1d4a96 field: swimlane.audit_log.category target_field: log.category ignore_missing: true - rename: + tag: rename_swimlane_audit_log_logFeatureCategory_to_log_feature_category_e1839923 field: swimlane.audit_log.logFeatureCategory target_field: log.feature_category ignore_missing: true - rename: + tag: rename_swimlane_audit_log_logSource_to_log_source_type_d85318e8 field: swimlane.audit_log.logSource target_field: log.source.type ignore_missing: true - rename: + tag: rename_swimlane_audit_log_logType_to_log_type_f49fe23a field: swimlane.audit_log.logType target_field: log.type ignore_missing: true - rename: + tag: rename_swimlane_audit_log_description_to_message_7e7bb22b field: swimlane.audit_log.description target_field: message ignore_missing: true - set: + tag: set_message_da12ebac field: message value: '{{{message}}}' if: ctx.message != null && ctx.message != '' - rename: + tag: rename_swimlane_audit_log_userId_to_user_id_aab25936 field: swimlane.audit_log.userId target_field: user.id ignore_missing: true @@ -145,47 +169,59 @@ processors: def hm = new HashMap(params.get(ctx.swimlane?.audit_log?.ActionType)); hm.forEach((k, v) -> ctx.event[k] = v); - rename: + tag: rename_swimlane_audit_log_category_to_log_category_fa1d4a96 field: swimlane.audit_log.category target_field: log.category ignore_missing: true - grok: + tag: grok_swimlane_audit_log_endpoint_2bf2105a field: swimlane.audit_log.endpoint patterns: - '%{GREEDYDATA:url.path}' ignore_missing: true - drop: + tag: drop_df5a2f3e description: Drop events related to the hubs/record endpoint if: ctx.url?.path instanceof String && ctx.url.path.toLowerCase().contains('hubs/record') - drop: + tag: drop_1abc01dd description: Drop events related to any record updates if: ctx.url?.path instanceof String && ctx.url.path.toLowerCase().endsWith('/values') - rename: + tag: rename_swimlane_audit_log_eventOutcome_to_event_outcome_b878d118 field: swimlane.audit_log.eventOutcome target_field: event.outcome ignore_missing: true - lowercase: + tag: lowercase_event_outcome_eb9dfc47 field: event.outcome ignore_missing: true - set: + tag: set_user_roles_558b689e field: user.roles value: ['administrator'] if: ctx.swimlane?.audit_log?.isAdmin == 'True' - rename: + tag: rename_swimlane_audit_log_authenticationType_to_user_authentication_type_b4a5d0bb field: swimlane.audit_log.authenticationType target_field: user.authentication.type ignore_missing: true - lowercase: + tag: lowercase_user_authentication_type_f3146340 field: user.authentication.type ignore_missing: true - user_agent: + tag: user_agent_swimlane_audit_log_userAgent_30bda3c1 field: swimlane.audit_log.userAgent ignore_missing: true - date: + tag: date_swimlane_audit_log_eventTime_to_timestamp_a7ebb4c6 field: swimlane.audit_log.eventTime target_field: '@timestamp' - formats: + formats: - ISO8601 - remove: + tag: remove_swimlane_96409c73 field: swimlane ignore_missing: false on_failure: diff --git a/packages/swimlane/data_stream/swimlane_api/elasticsearch/ingest_pipeline/default.yml b/packages/swimlane/data_stream/swimlane_api/elasticsearch/ingest_pipeline/default.yml index bc330e12691..e4d2e7d2d53 100644 --- a/packages/swimlane/data_stream/swimlane_api/elasticsearch/ingest_pipeline/default.yml +++ b/packages/swimlane/data_stream/swimlane_api/elasticsearch/ingest_pipeline/default.yml @@ -6,34 +6,43 @@ processors: tag: set_ecs_version value: 8.17.0 - set: + tag: set_event_original_b9758751 field: event.original copy_from: message - json: + tag: json_message_to_json_576c813b field: message target_field: json ignore_failure: true - drop: + tag: drop_631a5346 if: ctx.json == null || !(ctx.json instanceof Map) - remove: - field: + tag: remove_d1205855 + field: - message - input - stream - log ignore_missing: true - rename: + tag: rename_json_to_swimlane_audit_log_d2655548 field: json target_field: swimlane.audit_log ignore_missing: true - drop: + tag: drop_83b3b0cb if: ctx.swimlane?.audit_log?.LogType != 'Audit' - drop: + tag: drop_4f7f469b if: ctx.swimlane?.audit_log?.Description instanceof String && ctx.swimlane.audit_log.Description.toLowerCase().contains('read') - json: + tag: json_swimlane_audit_log_NewValue_7826aa37 field: swimlane.audit_log.NewValue if: ctx.swimlane.audit_log.NewValue != null ignore_failure: true - rename: + tag: rename_swimlane_audit_log_User_to_user_name_1343b4c1 field: swimlane.audit_log.User target_field: user.name ignore_missing: true @@ -50,77 +59,94 @@ processors: ignore_failure: true if: ctx.user?.name == null - append: + tag: append_related_user_3b423052 field: related.user value: "{{{user.name}}}" if: ctx.user?.name != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_36d4b55a field: related.user value: "{{{user.email}}}" if: ctx.user?.email != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_swimlane_audit_log_AccountId_to_cloud_origin_account_id_ad8c599d field: swimlane.audit_log.AccountId target_field: cloud.origin.account.id ignore_missing: true - rename: + tag: rename_swimlane_audit_log_TenantId_to_cloud_origin_project_id_8ac9beaa field: swimlane.audit_log.TenantId target_field: cloud.origin.project.id ignore_missing: true - rename: + tag: rename_swimlane_audit_log_LogLevel_to_log_level_62b87df8 field: swimlane.audit_log.LogLevel target_field: log.level ignore_missing: true - rename: + tag: rename_swimlane_audit_log_Message_to_message_f6c0c7aa field: swimlane.audit_log.Message target_field: message ignore_missing: true - rename: + tag: rename_swimlane_audit_log_Path_to_url_path_b51e086d field: swimlane.audit_log.Path target_field: url.path ignore_missing: true - grok: + tag: grok_swimlane_audit_log_SourceIp_b92053dc field: swimlane.audit_log.SourceIp patterns: - '::ffff:%{IPV4:source.ip}' - '%{IPV4:source.ip}' - - '%{IPV6:source.ip}' + - '%{IPV6:source.ip}' ignore_failure: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - rename: + tag: rename_swimlane_audit_log_Tenant_to_cloud_origin_tenant_id_7161a5d8 field: swimlane.audit_log.Tenant target_field: cloud.origin.tenant.id ignore_missing: true - rename: + tag: rename_swimlane_audit_log_LogCategory_to_log_category_d3f06aa6 field: swimlane.audit_log.LogCategory target_field: log.category ignore_missing: true - rename: + tag: rename_swimlane_audit_log_LogFeatureCategory_to_log_feature_category_ea354a43 field: swimlane.audit_log.LogFeatureCategory target_field: log.feature_category ignore_missing: true - rename: + tag: rename_swimlane_audit_log_LogSource_to_log_source_type_0bbc5288 field: swimlane.audit_log.LogSource target_field: log.source.type ignore_missing: true - rename: + tag: rename_swimlane_audit_log_LogType_to_log_type_186b70da field: swimlane.audit_log.LogType target_field: log.type ignore_missing: true - rename: + tag: rename_swimlane_audit_log_Description_to_message_3225068b field: swimlane.audit_log.Description target_field: message ignore_missing: true - set: + tag: set_message_79c51558 field: message value: '{{{message}}}: {{{swimlane.audit_log.NewValue.Name}}} with Id: {{{swimlane.audit_log.NewValue.Id}}}' if: ctx.message != null && ctx.message != '' && ctx.swimlane?.audit_log?.NewValue?.Name != null && ctx.swimlane.audit_log.NewValue.Id != null - rename: + tag: rename_swimlane_audit_log_UserId_to_user_id_d33a52d6 field: swimlane.audit_log.UserId target_field: user.id ignore_missing: true @@ -145,7 +171,7 @@ processors: - start Logout: type: - - end + - end source: >- ctx.event.kind = 'event'; ctx.event.type = ['info']; @@ -158,46 +184,58 @@ processors: def hm = new HashMap(params.get(ctx.swimlane?.audit_log?.ActionType)); hm.forEach((k, v) -> ctx.event[k] = v); - rename: + tag: rename_swimlane_audit_log_Category_to_log_category_2687cc76 field: swimlane.audit_log.Category target_field: log.category ignore_missing: true - rename: + tag: rename_swimlane_audit_log_Endpoint_to_url_path_d34b6cc7 field: swimlane.audit_log.Endpoint target_field: url.path ignore_missing: true - drop: + tag: drop_df5a2f3e description: Drop events related to the hubs/record endpoint if: ctx.url?.path instanceof String && ctx.url.path.toLowerCase().contains('hubs/record') - drop: + tag: drop_1abc01dd description: Drop events related to any record updates if: ctx.url?.path instanceof String && ctx.url.path.toLowerCase().endsWith('/values') - rename: + tag: rename_swimlane_audit_log_EventOutcome_to_event_outcome_e0920fb8 field: swimlane.audit_log.EventOutcome target_field: event.outcome ignore_missing: true - lowercase: + tag: lowercase_event_outcome_eb9dfc47 field: event.outcome ignore_missing: true - set: + tag: set_user_roles_558b689e field: user.roles value: ['administrator'] if: ctx.swimlane?.audit_log?.isAdmin == 'True' - rename: + tag: rename_swimlane_audit_log_AuthenticationType_to_user_authentication_type_df8c1c5b field: swimlane.audit_log.AuthenticationType target_field: user.authentication.type ignore_missing: true - lowercase: + tag: lowercase_user_authentication_type_f3146340 field: user.authentication.type ignore_missing: true - user_agent: + tag: user_agent_swimlane_audit_log_UserAgent_29d1ee21 field: swimlane.audit_log.UserAgent ignore_missing: true - date: + tag: date_swimlane_audit_log_EventTime_to_timestamp_f88048e6 field: swimlane.audit_log.EventTime target_field: '@timestamp' - formats: + formats: - ISO8601 - remove: + tag: remove_swimlane_96409c73 field: swimlane ignore_missing: false on_failure: diff --git a/packages/swimlane/data_stream/tenant_api/elasticsearch/ingest_pipeline/default.yml b/packages/swimlane/data_stream/tenant_api/elasticsearch/ingest_pipeline/default.yml index 49db6e28c37..34073db02eb 100644 --- a/packages/swimlane/data_stream/tenant_api/elasticsearch/ingest_pipeline/default.yml +++ b/packages/swimlane/data_stream/tenant_api/elasticsearch/ingest_pipeline/default.yml @@ -6,28 +6,35 @@ processors: tag: set_ecs_version value: 8.17.0 - set: + tag: set_event_original_b9758751 field: event.original copy_from: message - json: + tag: json_message_to_json_576c813b field: message target_field: json ignore_failure: true - drop: + tag: drop_631a5346 if: ctx.json == null || !(ctx.json instanceof Map) - remove: - field: + tag: remove_d1205855 + field: - message - input - stream - log ignore_missing: true - rename: + tag: rename_json_to_swimlane_audit_log_d2655548 field: json target_field: swimlane.audit_log ignore_missing: true - drop: + tag: drop_83b3b0cb if: ctx.swimlane?.audit_log?.LogType != 'Audit' - rename: + tag: rename_swimlane_audit_log_User_to_user_name_1343b4c1 field: swimlane.audit_log.User target_field: user.name ignore_missing: true @@ -44,73 +51,89 @@ processors: ignore_failure: true if: ctx.user?.name == null - append: + tag: append_related_user_3b423052 field: related.user value: "{{{user.name}}}" if: ctx.user?.name != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_36d4b55a field: related.user value: "{{{user.email}}}" if: ctx.user?.email != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_swimlane_audit_log_AccountId_to_cloud_origin_account_id_ad8c599d field: swimlane.audit_log.AccountId target_field: cloud.origin.account.id ignore_missing: true - rename: + tag: rename_swimlane_audit_log_TenantId_to_cloud_origin_project_id_8ac9beaa field: swimlane.audit_log.TenantId target_field: cloud.origin.project.id ignore_missing: true - rename: + tag: rename_swimlane_audit_log_LogLevel_to_log_level_62b87df8 field: swimlane.audit_log.LogLevel target_field: log.level ignore_missing: true - rename: + tag: rename_swimlane_audit_log_Message_to_message_f6c0c7aa field: swimlane.audit_log.Message target_field: message ignore_missing: true - rename: + tag: rename_swimlane_audit_log_Path_to_url_path_b51e086d field: swimlane.audit_log.Path target_field: url.path ignore_missing: true - grok: + tag: grok_swimlane_audit_log_SourceIp_b92053dc field: swimlane.audit_log.SourceIp patterns: - '::ffff:%{IPV4:source.ip}' - '%{IPV4:source.ip}' - - '%{IPV6:source.ip}' + - '%{IPV6:source.ip}' ignore_failure: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - rename: + tag: rename_swimlane_audit_log_Tenant_to_cloud_origin_tenant_id_7161a5d8 field: swimlane.audit_log.Tenant target_field: cloud.origin.tenant.id ignore_missing: true - rename: + tag: rename_swimlane_audit_log_LogCategory_to_log_category_d3f06aa6 field: swimlane.audit_log.LogCategory target_field: log.category ignore_missing: true - rename: + tag: rename_swimlane_audit_log_LogFeatureCategory_to_log_feature_category_ea354a43 field: swimlane.audit_log.LogFeatureCategory target_field: log.feature_category ignore_missing: true - rename: + tag: rename_swimlane_audit_log_LogSource_to_log_source_type_0bbc5288 field: swimlane.audit_log.LogSource target_field: log.source.type ignore_missing: true - rename: + tag: rename_swimlane_audit_log_LogType_to_log_type_186b70da field: swimlane.audit_log.LogType target_field: log.type ignore_missing: true - rename: + tag: rename_swimlane_audit_log_Description_to_message_3225068b field: swimlane.audit_log.Description target_field: message ignore_missing: true - rename: + tag: rename_swimlane_audit_log_UserId_to_user_id_d33a52d6 field: swimlane.audit_log.UserId target_field: user.id ignore_missing: true @@ -135,7 +158,7 @@ processors: - start Logout: type: - - end + - end source: >- ctx.event.kind = 'event'; ctx.event.type = ['info']; @@ -148,52 +171,65 @@ processors: def hm = new HashMap(params.get(ctx.swimlane?.audit_log?.ActionType)); hm.forEach((k, v) -> ctx.event[k] = v); - rename: + tag: rename_swimlane_audit_log_Category_to_log_category_2687cc76 field: swimlane.audit_log.Category target_field: log.category ignore_missing: true - grok: + tag: grok_swimlane_audit_log_EndPoint_b612c867 field: swimlane.audit_log.EndPoint patterns: - '%{HOSTNAME:url.domain}/%{GREEDYDATA:url.path}' - rename: + tag: rename_swimlane_audit_log_EventOutcome_to_event_outcome_e0920fb8 field: swimlane.audit_log.EventOutcome target_field: event.outcome ignore_missing: true - lowercase: + tag: lowercase_event_outcome_eb9dfc47 field: event.outcome ignore_missing: true - set: + tag: set_user_roles_558b689e field: user.roles value: ['administrator'] if: ctx.swimlane?.audit_log?.isAdmin == 'True' - rename: + tag: rename_swimlane_audit_log_AuthenticationType_to_user_authentication_type_df8c1c5b field: swimlane.audit_log.AuthenticationType target_field: user.authentication.type ignore_missing: true - lowercase: + tag: lowercase_user_authentication_type_f3146340 field: user.authentication.type ignore_missing: true - rename: + tag: rename_swimlane_audit_log_NewValue_Id_to_destination_user_changes_id_34eaa959 field: swimlane.audit_log.NewValue.Id target_field: destination.user.changes.id ignore_missing: true - rename: + tag: rename_swimlane_audit_log_NewValue_Name_to_destination_user_changes_name_802b39c1 field: swimlane.audit_log.NewValue.Name target_field: destination.user.changes.name ignore_missing: true - rename: + tag: rename_swimlane_audit_log_NewValue_ModifiedByUser_to_source_user_changes_id_242f1226 field: swimlane.audit_log.NewValue.ModifiedByUser target_field: source.user.changes.id - ignore_missing: true + ignore_missing: true - user_agent: + tag: user_agent_swimlane_audit_log_UserAgent_29d1ee21 field: swimlane.audit_log.UserAgent ignore_missing: true - date: + tag: date_swimlane_audit_log_EventTime_to_timestamp_f88048e6 field: swimlane.audit_log.EventTime target_field: '@timestamp' - formats: + formats: - ISO8601 - remove: + tag: remove_swimlane_96409c73 field: swimlane ignore_missing: false on_failure: diff --git a/packages/swimlane/data_stream/turbine_api/elasticsearch/ingest_pipeline/default.yml b/packages/swimlane/data_stream/turbine_api/elasticsearch/ingest_pipeline/default.yml index 2422714ccf6..f992317d8e8 100644 --- a/packages/swimlane/data_stream/turbine_api/elasticsearch/ingest_pipeline/default.yml +++ b/packages/swimlane/data_stream/turbine_api/elasticsearch/ingest_pipeline/default.yml @@ -6,33 +6,41 @@ processors: tag: set_ecs_version value: 8.17.0 - set: + tag: set_event_original_b9758751 field: event.original copy_from: message - json: + tag: json_message_to_json_576c813b field: message target_field: json ignore_failure: true - drop: + tag: drop_631a5346 if: ctx.json == null || !(ctx.json instanceof Map) - remove: - field: + tag: remove_d1205855 + field: - message - input - stream - log ignore_missing: true - rename: + tag: rename_json_to_swimlane_audit_log_d2655548 field: json target_field: swimlane.audit_log ignore_missing: true - drop: + tag: drop_83b3b0cb if: ctx.swimlane?.audit_log?.LogType != 'Audit' - json: + tag: json_swimlane_audit_log_NewValue_to_swimlane_audit_log_NewValue_9b3c2f0e field: swimlane.audit_log.NewValue target_field: swimlane.audit_log.NewValue if: ctx.swimlane?.audit_log?.NewValue != null ignore_failure: true - rename: + tag: rename_swimlane_audit_log_User_to_user_name_1343b4c1 field: swimlane.audit_log.User target_field: user.name ignore_missing: true @@ -49,34 +57,41 @@ processors: ignore_failure: true if: ctx.user?.name == null - append: + tag: append_related_user_3b423052 field: related.user value: "{{{user.name}}}" if: ctx.user?.name != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_36d4b55a field: related.user value: "{{{user.email}}}" if: ctx.user?.email != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_swimlane_audit_log_AccountId_to_cloud_origin_account_id_ad8c599d field: swimlane.audit_log.AccountId target_field: cloud.origin.account.id ignore_missing: true - rename: + tag: rename_swimlane_audit_log_TenantId_to_cloud_origin_project_id_8ac9beaa field: swimlane.audit_log.TenantId target_field: cloud.origin.project.id ignore_missing: true - rename: + tag: rename_swimlane_audit_log_LogLevel_to_log_level_62b87df8 field: swimlane.audit_log.LogLevel target_field: log.level ignore_missing: true - rename: + tag: rename_swimlane_audit_log_Message_to_message_f6c0c7aa field: swimlane.audit_log.Message target_field: message ignore_missing: true - script: + tag: script_eb7a89f0 lang: painless description: Normalize SourceIp to a single value source: >- @@ -89,45 +104,55 @@ processors: } } - grok: + tag: grok_swimlane_audit_log_SourceIp_b92053dc field: swimlane.audit_log.SourceIp patterns: - '::ffff:%{IPV4:source.ip}' - '%{IPV4:source.ip}' - - '%{IPV6:source.ip}' + - '%{IPV6:source.ip}' ignore_failure: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - rename: + tag: rename_swimlane_audit_log_Tenant_to_cloud_origin_tenant_id_7161a5d8 field: swimlane.audit_log.Tenant target_field: cloud.origin.tenant.id ignore_missing: true - rename: + tag: rename_swimlane_audit_log_LogCategory_to_log_category_d3f06aa6 field: swimlane.audit_log.LogCategory target_field: log.category ignore_missing: true - rename: + tag: rename_swimlane_audit_log_LogFeatureCategory_to_log_feature_category_ea354a43 field: swimlane.audit_log.LogFeatureCategory target_field: log.feature_category ignore_missing: true - rename: + tag: rename_swimlane_audit_log_LogSource_to_log_source_type_0bbc5288 field: swimlane.audit_log.LogSource target_field: log.source.type ignore_missing: true - rename: + tag: rename_swimlane_audit_log_LogType_to_log_type_186b70da field: swimlane.audit_log.LogType target_field: log.type ignore_missing: true - rename: + tag: rename_swimlane_audit_log_Description_to_message_3225068b field: swimlane.audit_log.Description target_field: message ignore_missing: true - set: + tag: set_message_481c9e18 field: message value: '{{{message}}}: {{{swimlane.audit_log.NewValue.name}}} with Id: {{{swimlane.audit_log.NewValue.id}}}' if: ctx.message != null && ctx.message != '' && ctx.swimlane?.audit_log?.NewValue?.name != null && ctx.swimlane.audit_log.NewValue.id != null - rename: + tag: rename_swimlane_audit_log_UserId_to_user_id_d33a52d6 field: swimlane.audit_log.UserId target_field: user.id ignore_missing: true @@ -152,7 +177,7 @@ processors: - start Logout: type: - - end + - end source: >- ctx.event.kind = 'event'; ctx.event.type = ['info']; @@ -165,52 +190,65 @@ processors: def hm = new HashMap(params.get(ctx.swimlane?.audit_log?.ActionType)); hm.forEach((k, v) -> ctx.event[k] = v); - rename: + tag: rename_swimlane_audit_log_Category_to_log_category_2687cc76 field: swimlane.audit_log.Category target_field: log.category ignore_missing: true - rename: + tag: rename_swimlane_audit_log_Endpoint_to_url_path_d34b6cc7 field: swimlane.audit_log.Endpoint target_field: url.path ignore_missing: true - rename: + tag: rename_swimlane_audit_log_EventOutcome_to_event_outcome_e0920fb8 field: swimlane.audit_log.EventOutcome target_field: event.outcome ignore_missing: true - lowercase: + tag: lowercase_event_outcome_eb9dfc47 field: event.outcome ignore_missing: true - set: + tag: set_user_roles_558b689e field: user.roles value: ['administrator'] if: ctx.swimlane?.audit_log?.isAdmin == 'True' - rename: + tag: rename_swimlane_audit_log_AuthenticationType_to_user_authentication_type_df8c1c5b field: swimlane.audit_log.AuthenticationType target_field: user.authentication.type ignore_missing: true - lowercase: + tag: lowercase_user_authentication_type_f3146340 field: user.authentication.type ignore_missing: true - rename: + tag: rename_swimlane_audit_log_NewValue_Id_to_destination_user_changes_id_34eaa959 field: swimlane.audit_log.NewValue.Id target_field: destination.user.changes.id ignore_missing: true - rename: + tag: rename_swimlane_audit_log_NewValue_Name_to_destination_user_changes_name_802b39c1 field: swimlane.audit_log.NewValue.Name target_field: destination.user.changes.name ignore_missing: true - rename: + tag: rename_swimlane_audit_log_NewValue_ModifiedByUser_to_source_user_changes_id_242f1226 field: swimlane.audit_log.NewValue.ModifiedByUser target_field: source.user.changes.id - ignore_missing: true + ignore_missing: true - user_agent: + tag: user_agent_swimlane_audit_log_UserAgent_29d1ee21 field: swimlane.audit_log.UserAgent ignore_missing: true - date: + tag: date_swimlane_audit_log_EventTime_to_timestamp_f88048e6 field: swimlane.audit_log.EventTime target_field: '@timestamp' - formats: + formats: - ISO8601 - remove: + tag: remove_swimlane_96409c73 field: swimlane ignore_missing: false on_failure: diff --git a/packages/swimlane/manifest.yml b/packages/swimlane/manifest.yml index d23638ceedc..72698b00ce6 100644 --- a/packages/swimlane/manifest.yml +++ b/packages/swimlane/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: swimlane title: "Swimlane Turbine" -version: 0.5.1 +version: 0.5.2 description: "Collect Swimlane Turbine Audit logs with Elastic Agent" type: integration categories: diff --git a/packages/ti_cyware_intel_exchange/changelog.yml b/packages/ti_cyware_intel_exchange/changelog.yml index 05eb2840ee5..aaa12b5df99 100644 --- a/packages/ti_cyware_intel_exchange/changelog.yml +++ b/packages/ti_cyware_intel_exchange/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.4.3" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "0.4.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/ti_cyware_intel_exchange/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml b/packages/ti_cyware_intel_exchange/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml index e4ef0685459..04c3546a204 100644 --- a/packages/ti_cyware_intel_exchange/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_cyware_intel_exchange/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml @@ -38,6 +38,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -79,6 +80,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f7751d3a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -105,6 +107,7 @@ processors: if: ctx.json?.created != null && ctx.json.created != '' on_failure: - append: + tag: append_error_message_6834a02a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -116,6 +119,7 @@ processors: if: ctx.json?.ctix_created != null && ctx.json.ctix_created != '' on_failure: - append: + tag: append_error_message_b5013287 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -132,6 +136,7 @@ processors: if: ctx.json?.ctix_modified != null && ctx.json.ctix_modified != '' on_failure: - append: + tag: append_error_message_b7af3172 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -147,6 +152,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d0710820 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -224,6 +230,7 @@ processors: if: ctx.json?.name != '' on_failure: - rename: + tag: rename_json_name_to_ti_cyware_intel_exchange_indicator_name_1eb51e7a field: json.name target_field: ti_cyware_intel_exchange.indicator.name ignore_missing: true @@ -241,6 +248,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_b42f62cb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -251,6 +259,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f06af911 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -261,6 +270,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_754615eb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -271,6 +281,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_061b5045 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -281,6 +292,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2ec4abd6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -291,6 +303,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_42249443 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -302,6 +315,7 @@ processors: if: ctx.json?.modified != null && ctx.json.modified != '' on_failure: - append: + tag: append_error_message_afda5729 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -323,6 +337,7 @@ processors: if: ctx.json?.sdo_name != '' on_failure: - rename: + tag: rename_json_sdo_name_to_ti_cyware_intel_exchange_indicator_sdo_name_5bd349d8 field: json.sdo_name target_field: ti_cyware_intel_exchange.indicator.sdo_name ignore_missing: true @@ -381,7 +396,7 @@ processors: ignore_missing: true - foreach: field: json.sources - tag: foreach_sources_first_seen + tag: foreach_json_sources_bb6d4acb if: ctx.json?.sources instanceof List processor: append: @@ -406,7 +421,7 @@ processors: ignore_missing: true - foreach: field: json.sources - tag: foreach_sources_last_seen + tag: foreach_json_sources_055df765 if: ctx.json?.sources instanceof List processor: append: @@ -492,6 +507,7 @@ processors: if: ctx.json?.valid_from != null && ctx.json.valid_from != '' on_failure: - append: + tag: append_error_message_8136d0b1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -503,6 +519,7 @@ processors: if: ctx.json?.valid_until != null && ctx.json.valid_until != '' on_failure: - append: + tag: append_error_message_dd7cdcef field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' ###################### @@ -539,6 +556,7 @@ processors: ctx.ti_cyware_intel_exchange.indicator.deleted_at = _tmp_deleted_at; on_failure: - append: + tag: append_error_message_1ae85c69 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -615,6 +633,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/ti_cyware_intel_exchange/manifest.yml b/packages/ti_cyware_intel_exchange/manifest.yml index ef97cecaeed..7a826d5cb7c 100644 --- a/packages/ti_cyware_intel_exchange/manifest.yml +++ b/packages/ti_cyware_intel_exchange/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: ti_cyware_intel_exchange title: Cyware Intel Exchange -version: 0.4.2 +version: 0.4.3 description: Collect logs from Cyware Intel Exchange with Elastic Agent. type: integration categories: ["security", "threat_intel"] diff --git a/packages/ti_greynoise/changelog.yml b/packages/ti_greynoise/changelog.yml index 13aa8952355..e5c58e82a0e 100644 --- a/packages/ti_greynoise/changelog.yml +++ b/packages/ti_greynoise/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.9.4" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "0.9.3" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/ti_greynoise/data_stream/ip/elasticsearch/ingest_pipeline/default.yml b/packages/ti_greynoise/data_stream/ip/elasticsearch/ingest_pipeline/default.yml index 4ea8ce92fec..0504a128aff 100644 --- a/packages/ti_greynoise/data_stream/ip/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_greynoise/data_stream/ip/elasticsearch/ingest_pipeline/default.yml @@ -42,6 +42,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - fingerprint: @@ -94,6 +95,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a8154e27 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -120,6 +122,7 @@ processors: if: ctx.json?.business_service_intelligence?.last_updated != null && ctx.json.business_service_intelligence.last_updated != '' on_failure: - append: + tag: append_error_message_34b0379d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -160,6 +163,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_609e00a8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -175,6 +179,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e634be79 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -186,6 +191,7 @@ processors: if: ctx.json?.internet_scanner_intelligence?.last_seen != null && ctx.json.internet_scanner_intelligence.last_seen != '' on_failure: - append: + tag: append_error_message_4d5bfd01 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -198,6 +204,7 @@ processors: if: ctx.json?.internet_scanner_intelligence?.last_seen_timestamp != null && ctx.json.internet_scanner_intelligence.last_seen_timestamp != '' on_failure: - append: + tag: append_error_message_22202546 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -216,6 +223,7 @@ processors: target_field: greynoise.ip.internet_scanner_intelligence.metadata.asn ignore_missing: true - gsub: + tag: gsub_greynoise_ip_internet_scanner_intelligence_metadata_asn_to_threat_indicator_as_number_a8b7373b field: greynoise.ip.internet_scanner_intelligence.metadata.asn target_field: threat.indicator.as.number pattern: 'AS' @@ -228,9 +236,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_98a16d98 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: + tag: remove_threat_indicator_as_number_80cd56a8 field: threat.indicator.as.number - rename: field: json.internet_scanner_intelligence.metadata.category @@ -245,6 +255,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_58bd188f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -310,9 +321,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ee717e58 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_internet_scanner_intelligence_tags_efbc068a field: json.internet_scanner_intelligence.tags if: ctx.json?.internet_scanner_intelligence?.tags instanceof List processor: @@ -322,6 +335,7 @@ processors: value: '{{{_ingest._value.name}}}' allow_duplicates: false - foreach: + tag: foreach_greynoise_ip_internet_scanner_intelligence_tag_names_5fd52255 field: greynoise.ip.internet_scanner_intelligence.tag.names if: ctx.greynoise?.ip?.internet_scanner_intelligence?.tag?.names instanceof List processor: @@ -338,6 +352,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_414373e4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -348,6 +363,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_223c1b15 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -423,6 +439,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/ti_greynoise/manifest.yml b/packages/ti_greynoise/manifest.yml index c682e3d227d..3a3b204e9c9 100644 --- a/packages/ti_greynoise/manifest.yml +++ b/packages/ti_greynoise/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: ti_greynoise title: GreyNoise -version: "0.9.3" +version: "0.9.4" description: Collect Threat Intelligence Indicators from GreyNoise using Elastic Agent, and perform enrichment on Elasticsearch by correlating Indicators of Compromise (IOCs). type: integration categories: diff --git a/packages/vectra_rux/changelog.yml b/packages/vectra_rux/changelog.yml index bf1f54aa181..c5bed0dd79c 100644 --- a/packages/vectra_rux/changelog.yml +++ b/packages/vectra_rux/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.5.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "0.5.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/vectra_rux/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/vectra_rux/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 06a441a8d45..6e1b22ca5ae 100644 --- a/packages/vectra_rux/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/vectra_rux/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -39,6 +39,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -130,6 +131,7 @@ processors: if: ctx.event?.action != '' on_failure: - append: + tag: append_error_message_ce1e3af4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - join: @@ -139,6 +141,7 @@ processors: if: ctx.event?.action != null on_failure: - append: + tag: append_error_message_f136491d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -160,6 +163,7 @@ processors: if: ctx.json?.event_timestamp != null && ctx.json.event_timestamp != '' on_failure: - append: + tag: append_error_message_64a7d6af field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -202,6 +206,7 @@ processors: if: ctx.json?.source_ip != '' on_failure: - append: + tag: append_error_message_31245381 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -210,10 +215,12 @@ processors: copy_from: vectra_rux.audit.source_ip ignore_empty_value: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -391,6 +398,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/vectra_rux/data_stream/detection_event/elasticsearch/ingest_pipeline/default.yml b/packages/vectra_rux/data_stream/detection_event/elasticsearch/ingest_pipeline/default.yml index 9b75b88b1b2..9c0df7ec9f8 100644 --- a/packages/vectra_rux/data_stream/detection_event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/vectra_rux/data_stream/detection_event/elasticsearch/ingest_pipeline/default.yml @@ -35,6 +35,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -82,6 +83,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c31ebc50 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -120,6 +122,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_067fe3ec field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -167,7 +170,7 @@ processors: tag: append_host_id_into_related_hosts value: '{{{host.id}}}' allow_duplicates: false - if: ctx.host?.id != null + if: ctx.host?.id != null - rename: field: json.entity_uid tag: rename_entity_uid @@ -182,6 +185,7 @@ processors: if: ctx.json?.event_timestamp != null && ctx.json.event_timestamp != '' on_failure: - append: + tag: append_error_message_ef18c781 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -201,6 +205,7 @@ processors: target_field: vectra_rux.detection_event.mitre ignore_missing: true - foreach: + tag: foreach_vectra_rux_detection_event_mitre_7a055e72 field: vectra_rux.detection_event.mitre if: ctx.vectra_rux?.detection_event?.mitre instanceof List ignore_failure: true @@ -218,6 +223,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_34e5fa7c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -238,6 +244,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_0186344f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -260,6 +267,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_53395579 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -328,6 +336,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/vectra_rux/data_stream/entity_event/elasticsearch/ingest_pipeline/default.yml b/packages/vectra_rux/data_stream/entity_event/elasticsearch/ingest_pipeline/default.yml index c90462ca450..8377c52e395 100644 --- a/packages/vectra_rux/data_stream/entity_event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/vectra_rux/data_stream/entity_event/elasticsearch/ingest_pipeline/default.yml @@ -35,6 +35,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -80,6 +81,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d1fdfb15 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -90,6 +92,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_fa02e0d5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -131,7 +134,7 @@ processors: tag: append_host_id_into_related_hosts value: '{{{host.id}}}' allow_duplicates: false - if: ctx.host?.id != null + if: ctx.host?.id != null - date: field: json.event_timestamp tag: date_event_timestamp @@ -141,6 +144,7 @@ processors: if: ctx.json?.event_timestamp != null && ctx.json.event_timestamp != '' on_failure: - append: + tag: append_error_message_a57ad9ed field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -162,6 +166,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8705a74b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -172,6 +177,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_1848ee77 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -206,6 +212,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_067fe3ec field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -279,6 +286,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_bbe378dc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -299,6 +307,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_639b6b30 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: @@ -348,6 +357,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/vectra_rux/data_stream/health/elasticsearch/ingest_pipeline/default.yml b/packages/vectra_rux/data_stream/health/elasticsearch/ingest_pipeline/default.yml index 5497564222d..9d818066369 100644 --- a/packages/vectra_rux/data_stream/health/elasticsearch/ingest_pipeline/default.yml +++ b/packages/vectra_rux/data_stream/health/elasticsearch/ingest_pipeline/default.yml @@ -47,6 +47,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -55,6 +56,7 @@ processors: target_field: vectra_rux.health.network.interfaces ignore_missing: true - foreach: + tag: foreach_json_connectivity_sensors_bb60514d field: json.connectivity.sensors if: ctx.json?.connectivity?.sensors instanceof List processor: @@ -71,6 +73,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_connectivity_sensors_885beff1 field: json.connectivity.sensors if: ctx.json?.connectivity?.sensors instanceof List processor: @@ -92,6 +95,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8e22c31a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -102,6 +106,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_43453a8d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -112,6 +117,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_6d950525 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -126,6 +132,7 @@ processors: if: ctx.json?.cpu?.updated_at != null && ctx.json.cpu.updated_at != '' on_failure: - append: + tag: append_error_message_9c1d5e39 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -136,6 +143,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_b1eda2c3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -195,6 +203,7 @@ processors: if: ctx.json?.detection?.updated_at != null && ctx.json.detection.updated_at != '' on_failure: - append: + tag: append_error_message_b71d3574 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -235,6 +244,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_66249cd8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -245,6 +255,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_bc337510 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -255,6 +266,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_286e9087 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -265,6 +277,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_97b7bb5f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -294,6 +307,7 @@ processors: if: ctx.json?.disk?.updated_at != null && ctx.json.disk.updated_at != '' on_failure: - append: + tag: append_error_message_41e4a47a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -305,6 +319,7 @@ processors: if: ctx.json?.event_timestamp != null && ctx.json.event_timestamp != '' on_failure: - append: + tag: append_error_message_a94eb2f3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -325,6 +340,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f3aecd30 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -335,6 +351,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_36479aed field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -345,6 +362,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_be67519d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -359,6 +377,7 @@ processors: if: ctx.json?.hostid?.updated_at != null && ctx.json.hostid.updated_at != '' on_failure: - append: + tag: append_error_message_841133ac field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -369,6 +388,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9d505463 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -379,6 +399,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d5a09987 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -393,6 +414,7 @@ processors: if: ctx.json?.memory?.updated_at != null && ctx.json.memory.updated_at != '' on_failure: - append: + tag: append_error_message_bc94db90 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -403,6 +425,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_995b2880 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -413,6 +436,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_1b3bbdd6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -432,6 +456,7 @@ processors: if: ctx.json?.network?.updated_at != null && ctx.json.network.updated_at != '' on_failure: - append: + tag: append_error_message_878ce00d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -442,9 +467,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_efb5f6b6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_network_vlans_vlan_ids_d37b9a1c field: json.network.vlans.vlan_ids if: ctx.json?.network?.vlans?.vlan_ids instanceof List processor: @@ -485,9 +512,11 @@ processors: if: ctx.json?.power?.updated_at != null && ctx.json.power.updated_at != '' on_failure: - append: + tag: append_error_message_dcefbc5a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_sensors_cc2adddd field: json.sensors if: ctx.json?.sensors instanceof List processor: @@ -503,6 +532,7 @@ processors: tag: set_sensors_headend_uri value: '{{{_ingest._value.headend_uri}}}' - foreach: + tag: foreach_json_sensors_8007c8b4 field: json.sensors if: ctx.json?.sensors instanceof List processor: @@ -519,6 +549,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_sensors_59fe3c47 field: json.sensors if: ctx.json?.sensors instanceof List processor: @@ -528,6 +559,7 @@ processors: value: '{{{_ingest._value.ip_address}}}' allow_duplicates: false - foreach: + tag: foreach_json_sensors_ea469ea7 field: json.sensors if: ctx.json?.sensors instanceof List processor: @@ -537,6 +569,7 @@ processors: value: '{{{_ingest._value.ip_address}}}' allow_duplicates: false - foreach: + tag: foreach_json_sensors_7846c061 field: json.sensors if: ctx.json?.sensors instanceof List processor: @@ -551,6 +584,7 @@ processors: field: _ingest._value.last_seen ignore_missing: true - foreach: + tag: foreach_json_sensors_abe59fe3 field: json.sensors if: ctx.json?.sensors instanceof List processor: @@ -560,6 +594,7 @@ processors: value: '{{{_ingest._value.name}}}' allow_duplicates: false - foreach: + tag: foreach_json_sensors_62a04cad field: json.sensors if: ctx.json?.sensors instanceof List processor: @@ -569,6 +604,7 @@ processors: value: '{{{_ingest._value.serial_number}}}' allow_duplicates: false - foreach: + tag: foreach_json_sensors_394710ab field: json.sensors if: ctx.json?.sensors instanceof List processor: @@ -585,6 +621,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_sensors_399086f9 field: json.sensors if: ctx.json?.sensors instanceof List processor: @@ -594,6 +631,7 @@ processors: value: '{{{_ingest._value.version}}}' allow_duplicates: false - foreach: + tag: foreach_json_sensors_be14c680 field: json.sensors if: ctx.json?.sensors instanceof List processor: @@ -651,6 +689,7 @@ processors: if: ctx.json?.system?.updated_at != null && ctx.json.system.updated_at != '' on_failure: - append: + tag: append_error_message_663b07f6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -666,6 +705,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_23701d3d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -677,6 +717,7 @@ processors: if: ctx.json?.system?.version?.gmt != null && ctx.json.system.version.gmt != '' on_failure: - append: + tag: append_error_message_d39962ff field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -691,6 +732,7 @@ processors: if: ctx.json?.system?.version?.last_update != null && ctx.json.system.version.last_update != '' on_failure: - append: + tag: append_error_message_7c4732f7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -702,6 +744,7 @@ processors: if: ctx.json?.system?.version?.last_update_utc != null && ctx.json.system.version.last_update_utc != '' on_failure: - append: + tag: append_error_message_5dcf4c28 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -735,6 +778,7 @@ processors: copy_from: vectra_rux.health.system.version.vm_type ignore_empty_value: true - foreach: + tag: foreach_json_trafficdrop_sensors_bcf04b56 field: json.trafficdrop.sensors if: ctx.json?.trafficdrop?.sensors instanceof List processor: @@ -751,6 +795,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_trafficdrop_sensors_19e83dbc field: json.trafficdrop.sensors if: ctx.json?.trafficdrop?.sensors instanceof List processor: @@ -776,9 +821,11 @@ processors: if: ctx.json?.trafficdrop?.updated_at != null && ctx.json.trafficdrop.updated_at != '' on_failure: - append: + tag: append_error_message_06f98d37 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_vectra_rux_health_sensors_9b375788 field: vectra_rux.health.sensors if: ctx.vectra_rux?.health?.sensors instanceof List processor: @@ -844,6 +891,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/vectra_rux/data_stream/lockdown/elasticsearch/ingest_pipeline/default.yml b/packages/vectra_rux/data_stream/lockdown/elasticsearch/ingest_pipeline/default.yml index 33f870364f3..433d6c71177 100644 --- a/packages/vectra_rux/data_stream/lockdown/elasticsearch/ingest_pipeline/default.yml +++ b/packages/vectra_rux/data_stream/lockdown/elasticsearch/ingest_pipeline/default.yml @@ -35,6 +35,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - fingerprint: @@ -72,6 +73,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_37a48ed7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -103,7 +105,7 @@ processors: tag: append_host_id_into_related_hosts value: '{{{host.id}}}' allow_duplicates: false - if: ctx.host?.id != null + if: ctx.host?.id != null - rename: field: json.entity_name tag: rename_entity_name @@ -148,6 +150,7 @@ processors: if: ctx.json?.lock_event_timestamp != null && ctx.json.lock_event_timestamp != '' on_failure: - append: + tag: append_error_message_3824055e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -190,6 +193,7 @@ processors: if: ctx.json?.unlock_event_timestamp != null && ctx.json.unlock_event_timestamp != '' on_failure: - append: + tag: append_error_message_c1d15d2b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -243,6 +247,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/vectra_rux/manifest.yml b/packages/vectra_rux/manifest.yml index 69b64f56719..6742c261c9a 100644 --- a/packages/vectra_rux/manifest.yml +++ b/packages/vectra_rux/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: vectra_rux title: "Vectra RUX" -version: 0.5.1 +version: 0.5.2 description: "Collect logs from Vectra RUX with Elastic Agent." type: integration categories: diff --git a/packages/withsecure_elements/changelog.yml b/packages/withsecure_elements/changelog.yml index 8e37030946b..049c55ee436 100644 --- a/packages/withsecure_elements/changelog.yml +++ b/packages/withsecure_elements/changelog.yml @@ -1,3 +1,8 @@ +- version: "0.3.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "0.3.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/withsecure_elements/data_stream/incidents/elasticsearch/ingest_pipeline/default.yml b/packages/withsecure_elements/data_stream/incidents/elasticsearch/ingest_pipeline/default.yml index e6c47eb0bfe..11fd6db8e3c 100644 --- a/packages/withsecure_elements/data_stream/incidents/elasticsearch/ingest_pipeline/default.yml +++ b/packages/withsecure_elements/data_stream/incidents/elasticsearch/ingest_pipeline/default.yml @@ -21,6 +21,7 @@ processors: if: ctx.event?.original != null && ctx.event.original instanceof String on_failure: - set: + tag: set_error_message_cebe6eaa field: error.message value: 'Failed to parse JSON from event.original' tag: json_decode @@ -72,7 +73,7 @@ processors: target_field: _id ignore_missing: true tag: generate_document_id - + # Map to ECS fields using copy_from - set: field: event.id @@ -107,7 +108,6 @@ processors: } else if (normalized == 'severe') { ctx.event.severity = 99; } - # Set timestamps - date: field: withsecure_elements.incidents.createdTimestamp @@ -119,6 +119,7 @@ processors: tag: set_timestamp on_failure: - append: + tag: append_error_message_54574230 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -143,6 +144,7 @@ processors: tag: set_event_end on_failure: - append: + tag: append_error_message_52da5d7b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' @@ -152,6 +154,7 @@ processors: tag: set_pipeline_error_into_event_kind if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/withsecure_elements/data_stream/security_events/elasticsearch/ingest_pipeline/default.yml b/packages/withsecure_elements/data_stream/security_events/elasticsearch/ingest_pipeline/default.yml index 81c35d149cc..22b5b5b6e83 100644 --- a/packages/withsecure_elements/data_stream/security_events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/withsecure_elements/data_stream/security_events/elasticsearch/ingest_pipeline/default.yml @@ -21,6 +21,7 @@ processors: if: ctx.event?.original != null && ctx.event.original instanceof String on_failure: - set: + tag: set_error_message_ed130b04 field: error.message value: 'Failed to parse JSON from event.original' tag: json_decode @@ -64,7 +65,7 @@ processors: target_field: _id ignore_missing: true tag: generate_document_id - + # Map to ECS fields using copy_from - set: field: event.id @@ -97,7 +98,6 @@ processors: } else if (normalized == 'critical') { ctx.event.severity = 99; } - # Set timestamps - date: field: withsecure_elements.security_events.serverTimestamp @@ -109,6 +109,7 @@ processors: tag: set_timestamp on_failure: - append: + tag: append_error_message_4512bd14 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -121,6 +122,7 @@ processors: tag: convert_persistence_timestamp on_failure: - append: + tag: append_error_message_62c01212 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -133,6 +135,7 @@ processors: tag: convert_client_timestamp on_failure: - append: + tag: append_error_message_ff605dc2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' @@ -151,6 +154,7 @@ processors: tag: set_pipeline_error_into_event_kind if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/withsecure_elements/manifest.yml b/packages/withsecure_elements/manifest.yml index a2a59da7c6b..706321a78cf 100644 --- a/packages/withsecure_elements/manifest.yml +++ b/packages/withsecure_elements/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.3.2" name: withsecure_elements title: "WithSecure Elements" -version: "0.3.1" +version: "0.3.2" source: license: "Elastic-2.0" description: "Ingest WithSecure Elements incidents and security events data" From e1483cc119b7f7d5479d68e70a6f67ef516e928e Mon Sep 17 00:00:00 2001 From: kcreddy Date: Thu, 6 Aug 2026 19:44:51 +0530 Subject: [PATCH 2/3] ssi: update changelog links to PR #20575 Update the link field in each new changelog entry from the tracking issue to the merged pull request. Updates #20558 --- packages/airlock_digital/changelog.yml | 2 +- packages/armis/changelog.yml | 2 +- packages/bitsight/changelog.yml | 2 +- packages/cyera/changelog.yml | 2 +- packages/entro/changelog.yml | 2 +- packages/extrahop/changelog.yml | 2 +- packages/neon_cyber/changelog.yml | 2 +- packages/nozomi_networks/changelog.yml | 2 +- packages/qualys_gav/changelog.yml | 2 +- packages/qualys_was/changelog.yml | 2 +- packages/swimlane/changelog.yml | 2 +- packages/ti_cyware_intel_exchange/changelog.yml | 2 +- packages/ti_greynoise/changelog.yml | 2 +- packages/vectra_rux/changelog.yml | 2 +- packages/withsecure_elements/changelog.yml | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/airlock_digital/changelog.yml b/packages/airlock_digital/changelog.yml index 63c39d6e5bf..68a08cc7a2b 100644 --- a/packages/airlock_digital/changelog.yml +++ b/packages/airlock_digital/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20575 - version: "0.5.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/armis/changelog.yml b/packages/armis/changelog.yml index e018ace14fe..8dfe714b364 100644 --- a/packages/armis/changelog.yml +++ b/packages/armis/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20575 - version: "0.6.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/bitsight/changelog.yml b/packages/bitsight/changelog.yml index b331b77d613..c4ab9f0a4ba 100644 --- a/packages/bitsight/changelog.yml +++ b/packages/bitsight/changelog.yml @@ -2,7 +2,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20575 - version: "0.4.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/cyera/changelog.yml b/packages/cyera/changelog.yml index 3a0f091a2dc..ba41f05984c 100644 --- a/packages/cyera/changelog.yml +++ b/packages/cyera/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20575 - version: "0.10.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/entro/changelog.yml b/packages/entro/changelog.yml index 00e6f370659..c4db5a4a40a 100644 --- a/packages/entro/changelog.yml +++ b/packages/entro/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20575 - version: "0.4.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/extrahop/changelog.yml b/packages/extrahop/changelog.yml index a18c4054ab8..2af6dbf4550 100644 --- a/packages/extrahop/changelog.yml +++ b/packages/extrahop/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20575 - version: "0.3.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/neon_cyber/changelog.yml b/packages/neon_cyber/changelog.yml index 82a8b06a73e..7f836ebc57b 100644 --- a/packages/neon_cyber/changelog.yml +++ b/packages/neon_cyber/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20575 - version: "0.2.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/nozomi_networks/changelog.yml b/packages/nozomi_networks/changelog.yml index c3a0e7772c1..881b8066411 100644 --- a/packages/nozomi_networks/changelog.yml +++ b/packages/nozomi_networks/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20575 - version: "0.3.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/qualys_gav/changelog.yml b/packages/qualys_gav/changelog.yml index 3dcb8527445..eeda1296f6d 100644 --- a/packages/qualys_gav/changelog.yml +++ b/packages/qualys_gav/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20575 - version: "0.9.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/qualys_was/changelog.yml b/packages/qualys_was/changelog.yml index c07b5c7bf7f..9e256eac4f3 100644 --- a/packages/qualys_was/changelog.yml +++ b/packages/qualys_was/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20575 - version: "0.5.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/swimlane/changelog.yml b/packages/swimlane/changelog.yml index 5714a17fc9e..547d43573d3 100644 --- a/packages/swimlane/changelog.yml +++ b/packages/swimlane/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20575 - version: "0.5.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/ti_cyware_intel_exchange/changelog.yml b/packages/ti_cyware_intel_exchange/changelog.yml index aaa12b5df99..ec866637fff 100644 --- a/packages/ti_cyware_intel_exchange/changelog.yml +++ b/packages/ti_cyware_intel_exchange/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20575 - version: "0.4.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/ti_greynoise/changelog.yml b/packages/ti_greynoise/changelog.yml index e5c58e82a0e..77a347ca92b 100644 --- a/packages/ti_greynoise/changelog.yml +++ b/packages/ti_greynoise/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20575 - version: "0.9.3" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/vectra_rux/changelog.yml b/packages/vectra_rux/changelog.yml index c5bed0dd79c..e05a16b3506 100644 --- a/packages/vectra_rux/changelog.yml +++ b/packages/vectra_rux/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20575 - version: "0.5.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/withsecure_elements/changelog.yml b/packages/withsecure_elements/changelog.yml index 049c55ee436..86c7abd9953 100644 --- a/packages/withsecure_elements/changelog.yml +++ b/packages/withsecure_elements/changelog.yml @@ -2,7 +2,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20575 - version: "0.3.1" changes: - description: Set agentless deployment mode `release` field to `ga`. From 5315c709013bd94ae4f8b4cac56b572c3715528b Mon Sep 17 00:00:00 2001 From: kcreddy Date: Fri, 7 Aug 2026 14:11:07 +0530 Subject: [PATCH 3/3] Rewrite tags to avoid renaming existing ones --- .../elasticsearch/ingest_pipeline/default.yml | 6 ++- .../elasticsearch/ingest_pipeline/default.yml | 2 + .../elasticsearch/ingest_pipeline/default.yml | 2 + .../elasticsearch/ingest_pipeline/default.yml | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 28 +++++++--- .../elasticsearch/ingest_pipeline/default.yml | 2 + .../elasticsearch/ingest_pipeline/default.yml | 2 + .../elasticsearch/ingest_pipeline/default.yml | 2 + .../elasticsearch/ingest_pipeline/default.yml | 2 + .../elasticsearch/ingest_pipeline/default.yml | 2 + .../elasticsearch/ingest_pipeline/default.yml | 6 ++- .../elasticsearch/ingest_pipeline/default.yml | 6 ++- .../elasticsearch/ingest_pipeline/default.yml | 3 ++ .../elasticsearch/ingest_pipeline/default.yml | 12 ++++- .../elasticsearch/ingest_pipeline/default.yml | 2 + .../elasticsearch/ingest_pipeline/default.yml | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 2 + .../elasticsearch/ingest_pipeline/default.yml | 4 ++ .../elasticsearch/ingest_pipeline/default.yml | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 2 + .../elasticsearch/ingest_pipeline/default.yml | 5 +- .../elasticsearch/ingest_pipeline/default.yml | 2 + .../elasticsearch/ingest_pipeline/default.yml | 2 + .../elasticsearch/ingest_pipeline/default.yml | 53 +++++++++++++++++-- .../elasticsearch/ingest_pipeline/default.yml | 10 ++-- .../pipeline_knowledge_base.yml | 14 ++--- packages/swimlane/changelog.yml | 3 ++ .../elasticsearch/ingest_pipeline/default.yml | 10 ++-- .../elasticsearch/ingest_pipeline/default.yml | 5 +- .../elasticsearch/ingest_pipeline/default.yml | 5 +- .../elasticsearch/ingest_pipeline/default.yml | 5 +- .../elasticsearch/ingest_pipeline/default.yml | 10 +++- .../elasticsearch/ingest_pipeline/default.yml | 2 + ...se-correlation_detection_rule-pipeline.yml | 23 +++++--- .../elasticsearch/ingest_pipeline/default.yml | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 13 ++++- .../elasticsearch/ingest_pipeline/default.yml | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 3 ++ .../elasticsearch/ingest_pipeline/default.yml | 3 ++ 42 files changed, 230 insertions(+), 53 deletions(-) diff --git a/packages/airlock_digital/data_stream/agent/elasticsearch/ingest_pipeline/default.yml b/packages/airlock_digital/data_stream/agent/elasticsearch/ingest_pipeline/default.yml index a020583dbc9..dc617a00528 100644 --- a/packages/airlock_digital/data_stream/agent/elasticsearch/ingest_pipeline/default.yml +++ b/packages/airlock_digital/data_stream/agent/elasticsearch/ingest_pipeline/default.yml @@ -274,14 +274,16 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_ip field: _ingest._value.ip ignore_missing: true - append: + tag: append_error_message field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: field: json.policy_details.commlist - tag: foreach_json_policy_details_commlist_761aaae4 + tag: foreach_policy_details_commlist_ip_761aaae4 if: ctx.json?.policy_details?.commlist instanceof List processor: append: @@ -695,6 +697,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message_3a6ad1cf field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -705,6 +708,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/airlock_digital/data_stream/execution_histories/elasticsearch/ingest_pipeline/default.yml b/packages/airlock_digital/data_stream/execution_histories/elasticsearch/ingest_pipeline/default.yml index 28f6711370b..7e7f47ff07c 100644 --- a/packages/airlock_digital/data_stream/execution_histories/elasticsearch/ingest_pipeline/default.yml +++ b/packages/airlock_digital/data_stream/execution_histories/elasticsearch/ingest_pipeline/default.yml @@ -377,6 +377,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -387,6 +388,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/airlock_digital/data_stream/server_activities/elasticsearch/ingest_pipeline/default.yml b/packages/airlock_digital/data_stream/server_activities/elasticsearch/ingest_pipeline/default.yml index 74313a2d64a..6a756357a78 100644 --- a/packages/airlock_digital/data_stream/server_activities/elasticsearch/ingest_pipeline/default.yml +++ b/packages/airlock_digital/data_stream/server_activities/elasticsearch/ingest_pipeline/default.yml @@ -189,6 +189,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -199,6 +200,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/armis/data_stream/alert/elasticsearch/ingest_pipeline/default.yml b/packages/armis/data_stream/alert/elasticsearch/ingest_pipeline/default.yml index 3721f45101d..a6f768c5ea8 100755 --- a/packages/armis/data_stream/alert/elasticsearch/ingest_pipeline/default.yml +++ b/packages/armis/data_stream/alert/elasticsearch/ingest_pipeline/default.yml @@ -134,7 +134,7 @@ processors: allow_duplicates: false - foreach: field: armis.alert.device_ids - tag: foreach_armis_alert_device_ids_4938258b + tag: foreach_deviceIds_4938258b if: ctx.armis?.alert?.device_ids instanceof List processor: append: @@ -332,6 +332,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -342,6 +343,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/armis/data_stream/device/elasticsearch/ingest_pipeline/default.yml b/packages/armis/data_stream/device/elasticsearch/ingest_pipeline/default.yml index 70c71029e4d..72f4e113736 100644 --- a/packages/armis/data_stream/device/elasticsearch/ingest_pipeline/default.yml +++ b/packages/armis/data_stream/device/elasticsearch/ingest_pipeline/default.yml @@ -117,14 +117,16 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_firstSeen field: _ingest._value.firstSeen ignore_missing: true - foreach: field: json.dataSources - tag: foreach_json_dataSources_44a528e5 + tag: foreach_dataSources_44a528e5 if: ctx.json?.dataSources instanceof List processor: foreach: + tag: foreach_ingest_value_instances field: _ingest._value.instances ignore_failure: true processor: @@ -138,14 +140,16 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_firstSeen_743481c2 field: _ingest._value.firstSeen ignore_missing: true - foreach: field: json.dataSources - tag: foreach_json_dataSources_80aabde7 + tag: foreach_dataSources_80aabde7 if: ctx.json?.dataSources instanceof List processor: foreach: + tag: foreach_ingest_value_instances_2129e376 field: _ingest._value.instances ignore_failure: true processor: @@ -159,11 +163,12 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_lastSeen field: _ingest._value.lastSeen ignore_missing: true - foreach: field: json.dataSources - tag: foreach_json_dataSources_b14257f2 + tag: foreach_dataSources_b14257f2 if: ctx.json?.dataSources instanceof List processor: date: @@ -176,14 +181,16 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_lastSeen_c3795701 field: _ingest._value.lastSeen ignore_missing: true - foreach: field: json.dataSources - tag: foreach_json_dataSources_23b1bbea + tag: foreach_dataSources_23b1bbea if: ctx.json?.dataSources instanceof List processor: foreach: + tag: foreach_ingest_value_instances_20f07e12 field: _ingest._value.instances ignore_failure: true processor: @@ -195,7 +202,7 @@ processors: ignore_missing: true - foreach: field: json.dataSources - tag: foreach_json_dataSources_17aa49c5 + tag: foreach_dataSources_17aa49c5 if: ctx.json?.dataSources instanceof List processor: remove: @@ -230,7 +237,7 @@ processors: value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.start - tag: set_event_start_62b39146 + tag: set_event_start_from_device_first_seen_62b39146 copy_from: armis.device.first_seen ignore_empty_value: true - convert: @@ -268,9 +275,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value field: _ingest._value ignore_missing: true - append: + tag: append_error_message field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -305,9 +314,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_2b6b117d field: _ingest._value ignore_missing: true - append: + tag: append_error_message_ed446288 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -381,6 +392,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_808486c0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -505,6 +517,7 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_creationTime field: _ingest._value.creationTime ignore_missing: true - foreach: @@ -533,6 +546,7 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_lastSeenTime field: _ingest._value.lastSeenTime ignore_missing: true - foreach: @@ -725,6 +739,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message_dfb15182 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -735,6 +750,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/armis/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml b/packages/armis/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml index 99e28bd2e7c..3452a423522 100755 --- a/packages/armis/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml +++ b/packages/armis/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml @@ -655,6 +655,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -665,6 +666,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/bitsight/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml b/packages/bitsight/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml index 5061be5545f..94abfc941b2 100644 --- a/packages/bitsight/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml +++ b/packages/bitsight/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml @@ -181,6 +181,7 @@ processors: on_failure: - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -191,6 +192,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/cyera/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/cyera/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index ee80c8c9aba..1d3f0f7ae66 100644 --- a/packages/cyera/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cyera/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -309,6 +309,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -319,6 +320,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/cyera/data_stream/classification/elasticsearch/ingest_pipeline/default.yml b/packages/cyera/data_stream/classification/elasticsearch/ingest_pipeline/default.yml index 17612e8c024..1b57b96c392 100644 --- a/packages/cyera/data_stream/classification/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cyera/data_stream/classification/elasticsearch/ingest_pipeline/default.yml @@ -242,6 +242,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -252,6 +253,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/cyera/data_stream/datastore/elasticsearch/ingest_pipeline/default.yml b/packages/cyera/data_stream/datastore/elasticsearch/ingest_pipeline/default.yml index c1e59c85ade..102b86c67e4 100644 --- a/packages/cyera/data_stream/datastore/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cyera/data_stream/datastore/elasticsearch/ingest_pipeline/default.yml @@ -622,6 +622,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -632,6 +633,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/cyera/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/cyera/data_stream/event/elasticsearch/ingest_pipeline/default.yml index f4bdd47fadb..e21ea4f9c08 100644 --- a/packages/cyera/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cyera/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -103,15 +103,17 @@ processors: processor: convert: field: _ingest._value.recordsCount.recordsCount - tag: convert_affectedDataClassAppearances_recordsCount_recordsCount _to_long + tag: convert_affectedDataClassAppearances_recordsCount_recordsCount_to_long target_field: _ingest._value.records_count.value type: long ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_recordsCount_recordsCount field: _ingest._value.recordsCount.recordsCount ignore_missing: true - append: + tag: append_error_message field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -676,6 +678,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message_c14f7af1 field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -686,6 +689,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/cyera/data_stream/issue/elasticsearch/ingest_pipeline/default.yml b/packages/cyera/data_stream/issue/elasticsearch/ingest_pipeline/default.yml index 3394e628f04..b5ff76a69be 100644 --- a/packages/cyera/data_stream/issue/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cyera/data_stream/issue/elasticsearch/ingest_pipeline/default.yml @@ -153,7 +153,7 @@ processors: ignore_missing: true - foreach: field: json.datastoreOwners - tag: foreach_json_datastoreOwners_b7acef9e + tag: foreach_datastoreOwners_datastoreOwnerUid_b7acef9e if: ctx.json?.datastoreOwners instanceof List processor: append: @@ -183,7 +183,7 @@ processors: ignore_missing: true - foreach: field: json.datastoreOwners - tag: foreach_json_datastoreOwners_920ad464 + tag: foreach_datastoreOwners_ownerType_920ad464 if: ctx.json?.datastoreOwners instanceof List processor: append: @@ -490,6 +490,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -500,6 +501,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/entro/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/entro/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index ad6d3077b6d..0ca7b232ce1 100644 --- a/packages/entro/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/entro/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -90,13 +90,16 @@ processors: ignore_missing: true on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' diff --git a/packages/extrahop/data_stream/detection/elasticsearch/ingest_pipeline/default.yml b/packages/extrahop/data_stream/detection/elasticsearch/ingest_pipeline/default.yml index a3529abb2ff..92e67cb7f26 100644 --- a/packages/extrahop/data_stream/detection/elasticsearch/ingest_pipeline/default.yml +++ b/packages/extrahop/data_stream/detection/elasticsearch/ingest_pipeline/default.yml @@ -241,9 +241,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_external field: _ingest._value.external ignore_missing: true - append: + tag: append_error_message field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -268,9 +270,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_object_id field: _ingest._value.object_id ignore_missing: true - append: + tag: append_error_message_f958f5ab field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -285,9 +289,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_object_id_98106835 field: _ingest._value.object_id ignore_missing: true - append: + tag: append_error_message_78c8443e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -302,14 +308,16 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_object_value field: _ingest._value.object_value ignore_missing: true - append: + tag: append_error_message_ea51762b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: field: json.participants - tag: foreach_json_participants_a474cd8b + tag: foreach_participants_object_value_a474cd8b if: ctx.json?.participants instanceof List processor: append: @@ -566,6 +574,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message_872cfe3d field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -576,6 +585,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/extrahop/data_stream/investigation/elasticsearch/ingest_pipeline/default.yml b/packages/extrahop/data_stream/investigation/elasticsearch/ingest_pipeline/default.yml index 2d0bdd106e1..b017f0d06fc 100644 --- a/packages/extrahop/data_stream/investigation/elasticsearch/ingest_pipeline/default.yml +++ b/packages/extrahop/data_stream/investigation/elasticsearch/ingest_pipeline/default.yml @@ -285,6 +285,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -295,6 +296,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/neon_cyber/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/neon_cyber/data_stream/events/elasticsearch/ingest_pipeline/default.yml index a9829c2c021..4c6fd0bfc4b 100644 --- a/packages/neon_cyber/data_stream/events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/neon_cyber/data_stream/events/elasticsearch/ingest_pipeline/default.yml @@ -69,7 +69,7 @@ processors: - date: field: json.event_timestamp target_field: '@timestamp' - tag: date_json_event_timestamp_to_timestamp_40752292 + tag: date_event_timestamp_40752292 formats: - ISO8601 on_failure: diff --git a/packages/nozomi_networks/data_stream/alert/elasticsearch/ingest_pipeline/default.yml b/packages/nozomi_networks/data_stream/alert/elasticsearch/ingest_pipeline/default.yml index 41ea8d6f546..d1b3a6b69cc 100644 --- a/packages/nozomi_networks/data_stream/alert/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nozomi_networks/data_stream/alert/elasticsearch/ingest_pipeline/default.yml @@ -875,6 +875,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -885,6 +886,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/nozomi_networks/data_stream/asset/elasticsearch/ingest_pipeline/default.yml b/packages/nozomi_networks/data_stream/asset/elasticsearch/ingest_pipeline/default.yml index ecd9a11f4d2..fe8f3502097 100644 --- a/packages/nozomi_networks/data_stream/asset/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nozomi_networks/data_stream/asset/elasticsearch/ingest_pipeline/default.yml @@ -215,8 +215,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value field: _ingest._value - append: + tag: append_error_message field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1021,6 +1023,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message_4575403f field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1031,6 +1034,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/nozomi_networks/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/nozomi_networks/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 4737fa34067..4776278d54f 100644 --- a/packages/nozomi_networks/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nozomi_networks/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -230,7 +230,7 @@ processors: if: ctx.user?.name != null - append: field: related.user - tag: append_related_user_fc96f0f9 + tag: append_audit_username_into_related_user_fc96f0f9 value: '{{{user.email}}}' allow_duplicates: false if: ctx.user?.email != null @@ -287,6 +287,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -297,6 +298,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/nozomi_networks/data_stream/health/elasticsearch/ingest_pipeline/default.yml b/packages/nozomi_networks/data_stream/health/elasticsearch/ingest_pipeline/default.yml index 7c8fa135857..113bc620065 100644 --- a/packages/nozomi_networks/data_stream/health/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nozomi_networks/data_stream/health/elasticsearch/ingest_pipeline/default.yml @@ -76,7 +76,7 @@ processors: ignore_empty_value: true - set: field: host.name - tag: set_host_name_c1844dfc + tag: set_host_hostname_from_health_appliance_host_c1844dfc copy_from: nozomi_networks.health.appliance_host ignore_empty_value: true - append: @@ -288,6 +288,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -298,6 +299,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/nozomi_networks/data_stream/node/elasticsearch/ingest_pipeline/default.yml b/packages/nozomi_networks/data_stream/node/elasticsearch/ingest_pipeline/default.yml index 059632e6246..f1ed3f6f939 100644 --- a/packages/nozomi_networks/data_stream/node/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nozomi_networks/data_stream/node/elasticsearch/ingest_pipeline/default.yml @@ -1072,6 +1072,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1082,6 +1083,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/nozomi_networks/data_stream/node_cve/elasticsearch/ingest_pipeline/default.yml b/packages/nozomi_networks/data_stream/node_cve/elasticsearch/ingest_pipeline/default.yml index 7ec87cf52d2..628caee5c36 100644 --- a/packages/nozomi_networks/data_stream/node_cve/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nozomi_networks/data_stream/node_cve/elasticsearch/ingest_pipeline/default.yml @@ -383,11 +383,12 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - remove: + tag: remove_ingest_value field: _ingest._value ignore_missing: true - foreach: field: json.nodes - tag: foreach_json_nodes_98ad5bbe + tag: foreach_nodes_98ad5bbe if: ctx.json?.nodes instanceof List processor: append: @@ -588,6 +589,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -598,6 +600,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/nozomi_networks/data_stream/session/elasticsearch/ingest_pipeline/default.yml b/packages/nozomi_networks/data_stream/session/elasticsearch/ingest_pipeline/default.yml index 7907c49322b..414511f58ad 100644 --- a/packages/nozomi_networks/data_stream/session/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nozomi_networks/data_stream/session/elasticsearch/ingest_pipeline/default.yml @@ -510,6 +510,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -520,6 +521,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/nozomi_networks/data_stream/variable/elasticsearch/ingest_pipeline/default.yml b/packages/nozomi_networks/data_stream/variable/elasticsearch/ingest_pipeline/default.yml index 931471e85d8..6c1eb92a60f 100644 --- a/packages/nozomi_networks/data_stream/variable/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nozomi_networks/data_stream/variable/elasticsearch/ingest_pipeline/default.yml @@ -542,6 +542,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -552,6 +553,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/qualys_gav/data_stream/asset/elasticsearch/ingest_pipeline/default.yml b/packages/qualys_gav/data_stream/asset/elasticsearch/ingest_pipeline/default.yml index 9422422510f..64ebb8953b0 100644 --- a/packages/qualys_gav/data_stream/asset/elasticsearch/ingest_pipeline/default.yml +++ b/packages/qualys_gav/data_stream/asset/elasticsearch/ingest_pipeline/default.yml @@ -772,6 +772,7 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_created field: _ingest._value.created ignore_missing: true - foreach: @@ -788,6 +789,7 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_last_updated field: _ingest._value.last_updated ignore_missing: true - date: @@ -885,6 +887,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -918,9 +921,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value field: _ingest._value ignore_missing: true - append: + tag: append_error_message_4411f124 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -930,7 +935,7 @@ processors: processor: foreach: field: _ingest._value.address_ip_v4 - tag: foreach_asset_network_interface_list_data_network_interface_address_ip_v4 + tag: foreach_asset_network_interface_list_data_network_interface_address_ip_v4_43ff8a28 ignore_failure: true processor: append: @@ -951,6 +956,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_143963e5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -984,6 +990,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_7d48ee26 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1003,9 +1010,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_dfc3b655 field: _ingest._value ignore_missing: true - append: + tag: append_error_message_e03cc8cb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1015,7 +1024,7 @@ processors: processor: foreach: field: _ingest._value.address_ip_v6 - tag: foreach_asset_network_interface_list_data_network_interface_address_ip_v6 + tag: foreach_asset_network_interface_list_data_network_interface_address_ip_v6_dfc35121 ignore_failure: true processor: append: @@ -1037,6 +1046,7 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_mac_vendor_intro_date field: _ingest._value.mac_vendor_intro_date ignore_missing: true - foreach: @@ -1051,9 +1061,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_port field: _ingest._value.port ignore_missing: true - append: + tag: append_error_message_d6899848 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1068,9 +1080,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_detection_score field: _ingest._value.detection_score ignore_missing: true - append: + tag: append_error_message_8da8184e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1087,6 +1101,7 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_first_found field: _ingest._value.first_found ignore_missing: true - foreach: @@ -1103,6 +1118,7 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_last_updated_dcfad241 field: _ingest._value.last_updated ignore_missing: true - set: @@ -1565,7 +1581,7 @@ processors: processor: append: field: package.type - tag: append_sofware_list_data_software_name_into_package_name + tag: append_sofware_list_data_software_name_into_package_name_f09d95a0 value: '{{{_ingest._value.software_type}}}' allow_duplicates: false - foreach: @@ -1580,9 +1596,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_is_ignored field: _ingest._value.is_ignored ignore_missing: true - append: + tag: append_error_message_5639bfa9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1619,6 +1637,7 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_install_date field: _ingest._value.install_date ignore_missing: true - foreach: @@ -1655,6 +1674,7 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_last_updated_48fae1a8 field: _ingest._value.last_updated ignore_missing: true - foreach: @@ -1671,6 +1691,7 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_last_use_date field: _ingest._value.last_use_date ignore_missing: true - foreach: @@ -1685,9 +1706,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_is_package field: _ingest._value.is_package ignore_missing: true - append: + tag: append_error_message_e8e14e4a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1702,9 +1725,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_is_package_component field: _ingest._value.is_package_component ignore_missing: true - append: + tag: append_error_message_a4ddfd07 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1731,6 +1756,7 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_lifecycle_ga_date field: _ingest._value.lifecycle.ga_date ignore_missing: true - foreach: @@ -1747,6 +1773,7 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_lifecycle_eol_date field: _ingest._value.lifecycle.eol_date ignore_missing: true - foreach: @@ -1763,6 +1790,7 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_lifecycle_eos_date field: _ingest._value.lifecycle.eos_date ignore_missing: true - foreach: @@ -1777,9 +1805,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_lifecycle_detection_score field: _ingest._value.lifecycle.detection_score ignore_missing: true - append: + tag: append_error_message_f6de18a6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1799,7 +1829,7 @@ processors: processor: append: field: package.license - tag: append_software_list_data_software_support_stage_desc + tag: append_software_list_data_software_support_stage_desc_bbaeff2d value: '{{{_ingest._value.license.category}}}' allow_duplicates: false - foreach: @@ -1814,9 +1844,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_authorization_detection_score field: _ingest._value.authorization_detection_score ignore_missing: true - append: + tag: append_error_message_f296c5ec field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1838,6 +1870,7 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_first_seen field: _ingest._value.first_seen ignore_missing: true - foreach: @@ -1859,6 +1892,7 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_last_seen field: _ingest._value.last_seen ignore_missing: true - foreach: @@ -1873,9 +1907,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_criticality_score field: _ingest._value.criticality_score ignore_missing: true - append: + tag: append_error_message_dfdfe4bf field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1953,9 +1989,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_free field: _ingest._value.free ignore_missing: true - append: + tag: append_error_message_8c49b7cd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1970,9 +2008,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_size field: _ingest._value.size ignore_missing: true - append: + tag: append_error_message_47b2d786 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1989,6 +2029,7 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_created_date field: _ingest._value.created_date ignore_missing: true - foreach: @@ -2005,6 +2046,7 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_expiration_date field: _ingest._value.expiration_date ignore_missing: true - foreach: @@ -2021,6 +2063,7 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_updated_date field: _ingest._value.updated_date ignore_missing: true - foreach: @@ -2230,6 +2273,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message_30479dff field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -2240,6 +2284,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/qualys_was/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml b/packages/qualys_was/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml index 5853418aef0..bf74cf70d6c 100644 --- a/packages/qualys_was/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml +++ b/packages/qualys_was/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml @@ -35,7 +35,7 @@ processors: field: ecs.version value: '8.16.0' - terminate: - tag: terminate_3c149bdd + tag: data_collection_error_3c149bdd if: ctx.error?.message != null && ctx.message == null && ctx.event?.original == null description: error message set and no data to process. - set: @@ -113,7 +113,7 @@ processors: value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: if: ctx.qualys_was?.vulnerability?.qid != null - tag: script_d94403e4 + tag: vul_id_is_long_d94403e4 lang: painless source: > if (ctx.qualys_was.vulnerability.qid instanceof String) { @@ -251,7 +251,7 @@ processors: ignore_missing: true - rename: field: json.detection.webApp.id - tag: rename_json_detection_webApp_id_to_qualys_was_vulnerability_web_app_id_de71119f + tag: rename_webApp_url_de71119f target_field: qualys_was.vulnerability.web_app.id ignore_missing: true - script: @@ -470,7 +470,7 @@ processors: - pipeline: name: '{{ IngestPipeline "pipeline_knowledge_base" }}' if: ctx.Finding != null && ctx.Finding.knowledge_base != null && ctx.Finding.knowledge_base.size() > 0 - tag: pipeline_693c5eae + tag: pipeline_knowledge_base_693c5eae ignore_missing_pipeline: true - append: field: event.kind @@ -488,6 +488,7 @@ processors: description: 'Keep copy of original event if there are errors' on_failure: - append: + tag: append_error_message field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' @@ -498,6 +499,7 @@ on_failure: description: 'Overwrite event.kind with pipeline_error for terminal failure' if: ctx.error?.message != null - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/qualys_was/data_stream/vulnerability/elasticsearch/ingest_pipeline/pipeline_knowledge_base.yml b/packages/qualys_was/data_stream/vulnerability/elasticsearch/ingest_pipeline/pipeline_knowledge_base.yml index 5b6a1adc614..76afbd4b22f 100644 --- a/packages/qualys_was/data_stream/vulnerability/elasticsearch/ingest_pipeline/pipeline_knowledge_base.yml +++ b/packages/qualys_was/data_stream/vulnerability/elasticsearch/ingest_pipeline/pipeline_knowledge_base.yml @@ -107,7 +107,7 @@ processors: ignore_missing: true - rename: field: json.knowledge_base.QID - tag: rename_json_knowledge_base_QID_to_qualys_was_vulnerability_knowledge_base_qid_35560f2c + tag: rename_QID_35560f2c target_field: qualys_was.vulnerability.knowledge_base.qid ignore_missing: true - rename: @@ -186,7 +186,7 @@ processors: if: ctx.json?.knowledge_base?.CVSS?.BASE instanceof String - rename: field: json.knowledge_base.CVSS.BASE - tag: rename_json_knowledge_base_CVSS_BASE_to_qualys_was_vulnerability_knowledge_base_cvss_base_obj_06e1b687 + tag: rename_CVSS_BASE_06e1b687 target_field: qualys_was.vulnerability.knowledge_base.cvss.base_obj ignore_missing: true if: ctx.json?.knowledge_base?.CVSS?.BASE instanceof Object @@ -379,7 +379,7 @@ processors: processor: rename: field: _ingest._value.SRC_NAME - tag: rename_CORRELATION_EXPLOITS_EXPLT_SRC_SRC_NAME + tag: rename_CORRELATION_EXPLOITS_EXPLT_SRC_SRC_NAME_8678befe target_field: _ingest._value.name ignore_missing: true - foreach: @@ -394,7 +394,7 @@ processors: processor: rename: field: _ingest._value.DESC - tag: rename_CORRELATION_EXPLOITS_EXPLT_SRC_EXPLT_LIST_EXPLT_DESC + tag: rename_CORRELATION_EXPLOITS_EXPLT_SRC_EXPLT_LIST_EXPLT_DESC_e877f799 target_field: _ingest._value.desc ignore_missing: true - foreach: @@ -409,7 +409,7 @@ processors: processor: rename: field: _ingest._value.LINK - tag: rename_CORRELATION_EXPLOITS_EXPLT_SRC_EXPLT_LIST_EXPLT_LINK + tag: rename_CORRELATION_EXPLOITS_EXPLT_SRC_EXPLT_LIST_EXPLT_LINK_c7f77dd5 target_field: _ingest._value.link ignore_missing: true - foreach: @@ -424,7 +424,7 @@ processors: processor: rename: field: _ingest._value.REF - tag: rename_CORRELATION_EXPLOITS_EXPLT_SRC_EXPLT_LIST_EXPLT_REF + tag: rename_CORRELATION_EXPLOITS_EXPLT_SRC_EXPLT_LIST_EXPLT_REF_1cec4812 target_field: _ingest._value.ref ignore_missing: true - foreach: @@ -817,8 +817,10 @@ on_failure: value: preserve_original_event allow_duplicates: false - append: + tag: append_error_message field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_event_kind field: event.kind value: pipeline_error diff --git a/packages/swimlane/changelog.yml b/packages/swimlane/changelog.yml index 547d43573d3..e935112f80c 100644 --- a/packages/swimlane/changelog.yml +++ b/packages/swimlane/changelog.yml @@ -4,6 +4,9 @@ - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement link: https://github.com/elastic/integrations/pull/20575 + - description: Remove a redundant no-op `rename` processor for `swimlane.audit_log.category` in the `audit_logs` data stream. + type: bugfix + link: https://github.com/elastic/integrations/pull/20575 - version: "0.5.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/swimlane/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml b/packages/swimlane/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml index cc97a4102ea..973f4e7b1a1 100644 --- a/packages/swimlane/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/swimlane/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml @@ -137,7 +137,7 @@ processors: ignore_missing: true - script: lang: painless - tag: Add ECS categorization + tag: add_ecs_categorization_5da2a2f7 params: Create: type: @@ -168,11 +168,6 @@ processors: } def hm = new HashMap(params.get(ctx.swimlane?.audit_log?.ActionType)); hm.forEach((k, v) -> ctx.event[k] = v); - - rename: - tag: rename_swimlane_audit_log_category_to_log_category_fa1d4a96 - field: swimlane.audit_log.category - target_field: log.category - ignore_missing: true - grok: tag: grok_swimlane_audit_log_endpoint_2bf2105a field: swimlane.audit_log.endpoint @@ -226,13 +221,16 @@ processors: ignore_missing: false on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' diff --git a/packages/swimlane/data_stream/swimlane_api/elasticsearch/ingest_pipeline/default.yml b/packages/swimlane/data_stream/swimlane_api/elasticsearch/ingest_pipeline/default.yml index e4d2e7d2d53..4bcb882e685 100644 --- a/packages/swimlane/data_stream/swimlane_api/elasticsearch/ingest_pipeline/default.yml +++ b/packages/swimlane/data_stream/swimlane_api/elasticsearch/ingest_pipeline/default.yml @@ -152,7 +152,7 @@ processors: ignore_missing: true - script: lang: painless - tag: Add ECS categorization + tag: add_ecs_categorization_bd0e2ba1 params: Create: type: @@ -240,13 +240,16 @@ processors: ignore_missing: false on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' diff --git a/packages/swimlane/data_stream/tenant_api/elasticsearch/ingest_pipeline/default.yml b/packages/swimlane/data_stream/tenant_api/elasticsearch/ingest_pipeline/default.yml index 34073db02eb..366410dcdd6 100644 --- a/packages/swimlane/data_stream/tenant_api/elasticsearch/ingest_pipeline/default.yml +++ b/packages/swimlane/data_stream/tenant_api/elasticsearch/ingest_pipeline/default.yml @@ -139,7 +139,7 @@ processors: ignore_missing: true - script: lang: painless - tag: Add ECS categorization + tag: add_ecs_categorization_c47f10e9 params: Create: type: @@ -234,13 +234,16 @@ processors: ignore_missing: false on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' diff --git a/packages/swimlane/data_stream/turbine_api/elasticsearch/ingest_pipeline/default.yml b/packages/swimlane/data_stream/turbine_api/elasticsearch/ingest_pipeline/default.yml index f992317d8e8..bab184151c8 100644 --- a/packages/swimlane/data_stream/turbine_api/elasticsearch/ingest_pipeline/default.yml +++ b/packages/swimlane/data_stream/turbine_api/elasticsearch/ingest_pipeline/default.yml @@ -158,7 +158,7 @@ processors: ignore_missing: true - script: lang: painless - tag: Add ECS categorization + tag: add_ecs_categorization_3e8b0d64 params: Create: type: @@ -253,13 +253,16 @@ processors: ignore_missing: false on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' diff --git a/packages/ti_cyware_intel_exchange/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml b/packages/ti_cyware_intel_exchange/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml index 04c3546a204..47c0ec1dd30 100644 --- a/packages/ti_cyware_intel_exchange/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_cyware_intel_exchange/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml @@ -392,11 +392,12 @@ processors: - UNIX on_failure: - remove: + tag: remove_ingest_value_first_seen field: _ingest._value.first_seen ignore_missing: true - foreach: field: json.sources - tag: foreach_json_sources_bb6d4acb + tag: foreach_sources_first_seen_bb6d4acb if: ctx.json?.sources instanceof List processor: append: @@ -417,11 +418,12 @@ processors: - UNIX on_failure: - remove: + tag: remove_ingest_value_last_seen field: _ingest._value.last_seen ignore_missing: true - foreach: field: json.sources - tag: foreach_json_sources_055df765 + tag: foreach_sources_last_seen_055df765 if: ctx.json?.sources instanceof List processor: append: @@ -451,9 +453,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_score field: _ingest._value.score ignore_missing: true - append: + tag: append_error_message field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -640,6 +644,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message_7d89407f field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -650,6 +655,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/ti_greynoise/data_stream/ip/elasticsearch/ingest_pipeline/default.yml b/packages/ti_greynoise/data_stream/ip/elasticsearch/ingest_pipeline/default.yml index 0504a128aff..6be4a9b703c 100644 --- a/packages/ti_greynoise/data_stream/ip/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_greynoise/data_stream/ip/elasticsearch/ingest_pipeline/default.yml @@ -446,6 +446,7 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -456,6 +457,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags_fbf50489 field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/ti_greynoise/elasticsearch/ingest_pipeline/ti_greynoise-correlation_detection_rule-pipeline.yml b/packages/ti_greynoise/elasticsearch/ingest_pipeline/ti_greynoise-correlation_detection_rule-pipeline.yml index bd283801c53..72582ec0841 100644 --- a/packages/ti_greynoise/elasticsearch/ingest_pipeline/ti_greynoise-correlation_detection_rule-pipeline.yml +++ b/packages/ti_greynoise/elasticsearch/ingest_pipeline/ti_greynoise-correlation_detection_rule-pipeline.yml @@ -2,6 +2,7 @@ description: Pipeline for processing detected IOC events from the user's environment. processors: - script: + tag: script lang: painless if: ctx.threat?.enrichments != null; description: Store the first 'matched' and 'indicator' object from the alert, as it comes from GreyNoise API. @@ -14,6 +15,7 @@ processors: } on_failure: - append: + tag: append_error_message field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' # Retain event.dataset and event.module from the matched source index in temp_fields, @@ -21,12 +23,12 @@ processors: # and are essential for navigating to the corresponding document. - rename: field: event.dataset - tag : keep_event_dataset_from_alert + tag: keep_event_dataset_from_alert target_field: temp_fields.dataset ignore_missing: true - rename: field: event.module - tag : keep_event_module_from_alert + tag: keep_event_module_from_alert target_field: temp_fields.module ignore_missing: true - script: @@ -48,6 +50,7 @@ processors: } on_failure: - append: + tag: append_error_message_c8631e7c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' # Define hardcoded values required for processing. @@ -86,7 +89,7 @@ processors: # Extract relevant fields from temp_fields and map them to threat.indicator. - rename: field: temp_fields.indicator - tag : rename_temp_fields_to_indicator + tag: rename_temp_fields_to_indicator target_field: greynoise.ip ignore_missing: true - set: @@ -172,25 +175,26 @@ processors: if: ctx.threat?.indicator?.ip != null && ctx.threat?.indicator?.ip != '' - rename: field: temp_fields.matched.atomic - tag : rename_temp_fields_to_matched_value + tag: rename_temp_fields_to_matched_value target_field: threat.indicator.matched.value ignore_missing: true - rename: field: temp_fields.matched.field - tag : rename_temp_fields_to_matched_field + tag: rename_temp_fields_to_matched_field target_field: threat.indicator.matched.field ignore_missing: true - rename: field: temp_fields.dataset - tag : rename_temp_fields_to_matched_field + tag: rename_temp_fields_to_matched_field_b298d715 target_field: threat.indicator.matched.dataset ignore_missing: true - rename: field: temp_fields.module - tag : rename_temp_fields_to_matched_field + tag: rename_temp_fields_to_matched_field_d974fcd0 target_field: threat.indicator.matched.module ignore_missing: true - gsub: + tag: gsub_greynoise_ip_internet_scanner_intelligence_metadata_asn_to_threat_indicator_as_number field: greynoise.ip.internet_scanner_intelligence.metadata.asn target_field: threat.indicator.as.number pattern: 'AS' @@ -203,9 +207,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_210030f4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_greynoise_ip_internet_scanner_intelligence_tag_names field: greynoise.ip.internet_scanner_intelligence.tag.names if: ctx.greynoise?.ip?.internet_scanner_intelligence?.tag?.names instanceof List processor: @@ -241,12 +247,14 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_3e20ead6 field: tags value: preserve_original_event allow_duplicates: false if: ctx.error?.message != null on_failure: - append: + tag: append_error_message_48b0633f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -254,6 +262,7 @@ on_failure: tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags_a71abfd4 field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/vectra_rux/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/vectra_rux/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 6e1b22ca5ae..dff59c07eb5 100644 --- a/packages/vectra_rux/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/vectra_rux/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -405,13 +405,15 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind - tag: set_pipeline_error_to_event_kind + tag: set_pipeline_error_to_event_kind_7292f93b value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/vectra_rux/data_stream/detection_event/elasticsearch/ingest_pipeline/default.yml b/packages/vectra_rux/data_stream/detection_event/elasticsearch/ingest_pipeline/default.yml index 9c0df7ec9f8..1f3e3da2b71 100644 --- a/packages/vectra_rux/data_stream/detection_event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/vectra_rux/data_stream/detection_event/elasticsearch/ingest_pipeline/default.yml @@ -343,13 +343,15 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind - tag: set_pipeline_error_to_event_kind + tag: set_pipeline_error_to_event_kind_e666f0e3 value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/vectra_rux/data_stream/entity_event/elasticsearch/ingest_pipeline/default.yml b/packages/vectra_rux/data_stream/entity_event/elasticsearch/ingest_pipeline/default.yml index 8377c52e395..2edb73b5269 100644 --- a/packages/vectra_rux/data_stream/entity_event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/vectra_rux/data_stream/entity_event/elasticsearch/ingest_pipeline/default.yml @@ -364,13 +364,15 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind - tag: set_pipeline_error_to_event_kind + tag: set_pipeline_error_to_event_kind_162fd445 value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/vectra_rux/data_stream/health/elasticsearch/ingest_pipeline/default.yml b/packages/vectra_rux/data_stream/health/elasticsearch/ingest_pipeline/default.yml index 9d818066369..103d74b66b9 100644 --- a/packages/vectra_rux/data_stream/health/elasticsearch/ingest_pipeline/default.yml +++ b/packages/vectra_rux/data_stream/health/elasticsearch/ingest_pipeline/default.yml @@ -67,9 +67,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_ip_address field: _ingest._value.ip_address ignore_missing: true - append: + tag: append_error_message field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -543,9 +545,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_ip_address_ac3230f5 field: _ingest._value.ip_address ignore_missing: true - append: + tag: append_error_message_263fb6e9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -581,6 +585,7 @@ processors: - ISO8601 on_failure: - remove: + tag: remove_ingest_value_last_seen field: _ingest._value.last_seen ignore_missing: true - foreach: @@ -615,9 +620,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_update_count field: _ingest._value.update_count ignore_missing: true - append: + tag: append_error_message_def17ce7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -789,9 +796,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_ip_address_f675b353 field: _ingest._value.ip_address ignore_missing: true - append: + tag: append_error_message_138b9be5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -898,13 +907,15 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message_7c7a2933 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind - tag: set_pipeline_error_to_event_kind + tag: set_pipeline_error_to_event_kind_68926a8f value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/vectra_rux/data_stream/lockdown/elasticsearch/ingest_pipeline/default.yml b/packages/vectra_rux/data_stream/lockdown/elasticsearch/ingest_pipeline/default.yml index 433d6c71177..83e8512c3dc 100644 --- a/packages/vectra_rux/data_stream/lockdown/elasticsearch/ingest_pipeline/default.yml +++ b/packages/vectra_rux/data_stream/lockdown/elasticsearch/ingest_pipeline/default.yml @@ -254,13 +254,15 @@ processors: if: ctx.error?.message != null on_failure: - append: + tag: append_error_message field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind - tag: set_pipeline_error_to_event_kind + tag: set_pipeline_error_to_event_kind_147ffe6a value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/withsecure_elements/data_stream/incidents/elasticsearch/ingest_pipeline/default.yml b/packages/withsecure_elements/data_stream/incidents/elasticsearch/ingest_pipeline/default.yml index 11fd6db8e3c..976954cc8b2 100644 --- a/packages/withsecure_elements/data_stream/incidents/elasticsearch/ingest_pipeline/default.yml +++ b/packages/withsecure_elements/data_stream/incidents/elasticsearch/ingest_pipeline/default.yml @@ -161,15 +161,18 @@ processors: if: ctx.error?.message != null on_failure: - set: + tag: set_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' - set: + tag: set_event_kind_9da4dce5 field: event.kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/withsecure_elements/data_stream/security_events/elasticsearch/ingest_pipeline/default.yml b/packages/withsecure_elements/data_stream/security_events/elasticsearch/ingest_pipeline/default.yml index 22b5b5b6e83..e67a702ee84 100644 --- a/packages/withsecure_elements/data_stream/security_events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/withsecure_elements/data_stream/security_events/elasticsearch/ingest_pipeline/default.yml @@ -161,15 +161,18 @@ processors: if: ctx.error?.message != null on_failure: - set: + tag: set_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' - set: + tag: set_event_kind_0dea3c49 field: event.kind value: pipeline_error - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false