ssi: add processor tags and preserve_original_event on failure (4/4)

packages/airlock_digital/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.5.2"
+  changes:
+    - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/20575
 - version: "0.5.1"
   changes:
     - description: Set agentless deployment mode `release` field to `ga`.

packages/airlock_digital/data_stream/agent/elasticsearch/ingest_pipeline/default.yml

@@ -92,6 +92,7 @@ processors:
       ignore_missing: true
       on_failure:
         - append:
+            tag: append_error_message_ed15c765
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - rename:
@@ -129,6 +130,7 @@ processors:
       if: ctx.json?.ip != ''
       on_failure:
         - append:
+            tag: append_error_message_b1bdbb98
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - append:
@@ -152,6 +154,7 @@ processors:
       if: ctx.json?.lastcheckin != null && ctx.json.lastcheckin != ''
       on_failure:
         - append:
+            tag: append_error_message_1155faed
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - convert:
@@ -163,6 +166,7 @@ processors:
       if: ctx.json?.localip != ''
       on_failure:
         - append:
+            tag: append_error_message_d86b373c
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - append:
@@ -270,14 +274,16 @@ processors:
           ignore_missing: true
           on_failure:
             - remove:
+                tag: remove_ingest_value_ip
                 field: _ingest._value.ip
                 ignore_missing: true
             - append:
+                tag: append_error_message
                 field: error.message
                 value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - foreach:
       field: json.policy_details.commlist
-      tag: foreach_policy_details_commlist_ip
+      tag: foreach_policy_details_commlist_ip_761aaae4
       if: ctx.json?.policy_details?.commlist instanceof List
       processor:
         append:
@@ -462,6 +468,7 @@ processors:
       ignore_missing: true
       on_failure:
         - append:
+            tag: append_error_message_80c7c540
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - rename:
@@ -540,6 +547,7 @@ processors:
       ignore_missing: true
       on_failure:
         - append:
+            tag: append_error_message_97fa019b
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - convert:
@@ -583,6 +591,7 @@ processors:
       type: string
       ignore_missing: true
   - script:
+      tag: script_1fd0624f
       description: Map status to corresponding agent status.
       lang: painless
       params:
@@ -681,12 +690,14 @@ processors:
       value: pipeline_error
       if: ctx.error?.message != null
   - append:
+      tag: append_tags_9fe66b2c
       field: tags
       value: preserve_original_event
       allow_duplicates: false
       if: ctx.error?.message != null
 on_failure:
   - append:
+      tag: append_error_message_3a6ad1cf
       field: error.message
       value: |-
         Processor '{{{ _ingest.on_failure_processor_type }}}'
@@ -697,6 +708,7 @@ on_failure:
       tag: set_pipeline_error_to_event_kind
       value: pipeline_error
   - append:
+      tag: append_tags
       field: tags
       value: preserve_original_event
       allow_duplicates: false

packages/airlock_digital/data_stream/execution_histories/elasticsearch/ingest_pipeline/default.yml

@@ -38,6 +38,7 @@ processors:
       target_field: json
       on_failure:
         - append:
+            tag: append_error_message_cea45f41
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - append:
@@ -81,6 +82,7 @@ processors:
       if: ctx.json?.datetime != null && ctx.json.datetime != ''
       on_failure:
         - append:
+            tag: append_error_message_bf22f8e7
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - set:
@@ -99,11 +101,13 @@ processors:
       copy_from: airlock_digital.execution_histories.filename
       ignore_empty_value: true
   - gsub:
+      tag: gsub_file_path_517235af
       field: file.path
       pattern: '\\'
       replacement: '/'
       ignore_failure: true
   - split:
+      tag: split_file_path_to_file_name_19cfdeb5
       field: file.path
       separator: '/'
       target_field: file.name
@@ -265,9 +269,11 @@ processors:
       ignore_missing: true
       on_failure:
         - append:
+            tag: append_error_message_d064192d
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - script:
+      tag: script_5a14c280
       description: Map type to corresponding execution type.
       lang: painless
       params:
@@ -364,12 +370,14 @@ processors:
       value: pipeline_error
       if: ctx.error?.message != null
   - append:
+      tag: append_tags_9fe66b2c
       field: tags
       value: preserve_original_event
       allow_duplicates: false
       if: ctx.error?.message != null
 on_failure:
   - append:
+      tag: append_error_message
       field: error.message
       value: |-
         Processor '{{{ _ingest.on_failure_processor_type }}}'
@@ -380,6 +388,7 @@ on_failure:
       tag: set_pipeline_error_to_event_kind
       value: pipeline_error
   - append:
+      tag: append_tags
       field: tags
       value: preserve_original_event
       allow_duplicates: false

packages/airlock_digital/data_stream/server_activities/elasticsearch/ingest_pipeline/default.yml

@@ -68,6 +68,7 @@ processors:
       if: ctx.json?.datetime != null && ctx.json.datetime != ''
       on_failure:
         - append:
+            tag: append_error_message_63229d47
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - set:
@@ -107,6 +108,7 @@ processors:
       if: ctx.event?.action != ''
       on_failure:
         - append:
+            tag: append_error_message_ce1e3af4
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - join:
@@ -116,6 +118,7 @@ processors:
       if: ctx.event?.action != ''
       on_failure:
         - append:
+            tag: append_error_message_abbd6eec
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - rename:
@@ -179,12 +182,14 @@ processors:
       value: pipeline_error
       if: ctx.error?.message != null
   - append:
+      tag: append_tags_9fe66b2c
       field: tags
       value: preserve_original_event
       allow_duplicates: false
       if: ctx.error?.message != null
 on_failure:
   - append:
+      tag: append_error_message
       field: error.message
       value: |-
         Processor '{{{ _ingest.on_failure_processor_type }}}'
@@ -195,6 +200,7 @@ on_failure:
       tag: set_pipeline_error_to_event_kind
       value: pipeline_error
   - append:
+      tag: append_tags
       field: tags
       value: preserve_original_event
       allow_duplicates: false

packages/airlock_digital/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.3.2
 name: airlock_digital
 title: Airlock Digital
-version: 0.5.1
+version: 0.5.2
 description: Collect logs from Airlock Digital with Elastic Agent.
 type: integration
 categories:

packages/armis/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.6.2"
+  changes:
+    - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/20575
 - version: "0.6.1"
   changes:
     - description: Set agentless deployment mode `release` field to `ga`.

packages/armis/data_stream/alert/elasticsearch/ingest_pipeline/default.yml

@@ -42,6 +42,7 @@ processors:
       target_field: json
       on_failure:
         - append:
+            tag: append_error_message_cea45f41
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - fingerprint:
@@ -76,6 +77,7 @@ processors:
       ignore_missing: true
       on_failure:
         - append:
+            tag: append_error_message_6bf4e364
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - convert:
@@ -132,7 +134,7 @@ processors:
           allow_duplicates: false
   - foreach:
       field: armis.alert.device_ids
-      tag: foreach_deviceIds
+      tag: foreach_deviceIds_4938258b
       if: ctx.armis?.alert?.device_ids instanceof List
       processor:
         append:
@@ -151,6 +153,7 @@ processors:
       if: ctx.json?.lastAlertUpdateTime != null && ctx.json.lastAlertUpdateTime != ''
       on_failure:
         - append:
+            tag: append_error_message_962c6444
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - rename:
@@ -218,6 +221,7 @@ processors:
         }
       on_failure:
         - append:
+            tag: append_error_message_b608e76d
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - rename:
@@ -241,6 +245,7 @@ processors:
       if: ctx.json?.statusChangeTime != null && ctx.json.statusChangeTime != ''
       on_failure:
         - append:
+            tag: append_error_message_9981e61c
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - set:
@@ -259,6 +264,7 @@ processors:
       if: ctx.json?.time != null && ctx.json.time != ''
       on_failure:
         - append:
+            tag: append_error_message_7ab96520
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - rename:
@@ -319,12 +325,14 @@ processors:
       value: pipeline_error
       if: ctx.error?.message != null
   - append:
+      tag: append_tags_9fe66b2c
       field: tags
       value: preserve_original_event
       allow_duplicates: false
       if: ctx.error?.message != null
 on_failure:
   - append:
+      tag: append_error_message
       field: error.message
       value: >-
         Processor '{{{ _ingest.on_failure_processor_type }}}'
@@ -335,6 +343,7 @@ on_failure:
       tag: set_pipeline_error_to_event_kind
       value: pipeline_error
   - append:
+      tag: append_tags
       field: tags
       value: preserve_original_event
       allow_duplicates: false