ssi: add processor tags and preserve_original_event on failure (2/4)

packages/entityanalytics_okta/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.4.1"
+  changes:
+    - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/20573
 - version: "3.4.0"
   changes:
     - description: Add the Advanced Analytics (UEBA) subcategory
@@ -97,12 +102,12 @@
       link: https://github.com/elastic/integrations/pull/12206
 - version: "1.7.0"
   changes:
     - description: Add "preserve_original_event" tag to documents with `event.kind` manually set to "pipeline_error".
       type: enhancement
       link: https://github.com/elastic/integrations/pull/12109
 - version: "1.6.0"
   changes:
     - description: Add "preserve_original_event" tag to documents with `event.kind` set to "pipeline_error".
       type: enhancement
       link: https://github.com/elastic/integrations/pull/12046
 - version: "1.5.1"

packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/default.yml

@@ -7,6 +7,7 @@ processors:
       value: 8.11.0
 
   - remove:
+      tag: remove_event_action_9b3969af
       field: event.action
       if: ctx.event?.action != "started" && ctx.event?.action != "completed"
       ignore_missing: true
@@ -24,11 +25,13 @@ processors:
       tag: pipeline_to_device
       if: ctx.device?.id != null
   - rename:
+      tag: rename_okta_domain_to_host_name_bc4fe597
       field: okta_domain
       target_field: host.name
       ignore_missing: true
       on_failure:
         - rename:
+            tag: rename_okta_domain_to_okta_okta_domain_79b460fb
             field: okta_domain
             target_field: okta.okta_domain
             ignore_failure: true
@@ -57,6 +60,7 @@ processors:
       value: pipeline_error
       if: ctx.error?.message != null
   - append:
+      tag: append_tags_9fe66b2c
       field: tags
       value: preserve_original_event
       allow_duplicates: false

packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/device.yml

@@ -15,6 +15,7 @@ processors:
           ctx.event.original = stringified_orig;
         }
   - remove:
+      tag: remove_event_action_9b3969af
       field: event.action
       if: ctx.event?.action != "started" && ctx.event?.action != "completed"
       ignore_missing: true
@@ -72,8 +73,10 @@ processors:
       if: ctx.okta?.created != null && ctx.okta.created != ''
       on_failure:
         - remove:
+            tag: remove_okta_created_235a502d
             field: okta.created
         - append:
+            tag: append_error_message_729fae1b
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - set:
@@ -90,8 +93,10 @@ processors:
       if: ctx.okta?.activated != null && ctx.okta.activated != ''
       on_failure:
         - remove:
+            tag: remove_okta_activated_064816bc
             field: okta.activated
         - append:
+            tag: append_error_message_66f96d3f
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - date:
@@ -103,8 +108,10 @@ processors:
       if: ctx.okta?.statusChanged != null && ctx.okta.statusChanged != ''
       on_failure:
         - remove:
+            tag: remove_okta_statusChanged_01e67889
             field: okta.statusChanged
         - append:
+            tag: append_error_message_bd1a4715
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - set:
@@ -121,8 +128,10 @@ processors:
       if: ctx.okta?.lastUpdated != null && ctx.okta.lastUpdated != ''
       on_failure:
         - remove:
+            tag: remove_okta_lastUpdated_62fbc0a4
             field: okta.lastUpdated
         - append:
+            tag: append_error_message_97e5b363
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - set:
@@ -207,7 +216,7 @@ processors:
   - rename:
       field: okta.profile.diskEncryptionType
       target_field: entityanalytics_okta.device.profile.disk_encryption_type
-      tag: rename_device_profile_display_name
+      tag: rename_okta_profile_diskEncryptionType_to_entityanalytics_okta_device_profile_disk_encryption_type_8161cdc0
       ignore_missing: true
   - convert:
       field: okta.profile.registered
@@ -217,8 +226,10 @@ processors:
       ignore_missing: true
       on_failure:
         - remove:
+            tag: remove_okta_profile_registered_cd970fe3
             field: okta.profile.registered
         - append:
+            tag: append_error_message_5aff211e
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - convert:
@@ -229,8 +240,10 @@ processors:
       ignore_missing: true
       on_failure:
         - remove:
+            tag: remove_okta_profile_secure_hardware_present_a53eaf05
             field: okta.profile.secure_hardware_present
         - append:
+            tag: append_error_message_703b70b0
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - convert:
@@ -241,8 +254,10 @@ processors:
       ignore_missing: true
       on_failure:
         - remove:
+            tag: remove_okta_profile_managed_9556352a
             field: okta.profile.managed
         - append:
+            tag: append_error_message_3190ba2a
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - set:

packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/user.yml

@@ -15,6 +15,7 @@ processors:
           ctx.event.original = stringified_orig;
         }
   - remove:
+      tag: remove_event_action_9b3969af
       field: event.action
       if: ctx.event?.action != "started" && ctx.event?.action != "completed"
       ignore_missing: true
@@ -29,7 +30,7 @@ processors:
   - set:
       field: event.type
       tag: set_event_type
-      value: ['user','info']
+      value: ['user', 'info']
   - set:
       field: asset.category
       tag: set_asset_category
@@ -80,6 +81,7 @@ processors:
         }
       on_failure:
         - append:
+            tag: append_error_message_1c58f509
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - rename:
@@ -106,8 +108,10 @@ processors:
       if: ctx.okta?.created != null && ctx.okta.created != ''
       on_failure:
         - remove:
+            tag: remove_okta_created_400119ef
             field: okta.created
         - append:
+            tag: append_error_message_5d1ccd5d
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - set:
@@ -129,8 +133,10 @@ processors:
       if: ctx.okta?.activated != null && ctx.okta.activated != ''
       on_failure:
         - remove:
+            tag: remove_okta_activated_69d3eb42
             field: okta.activated
         - append:
+            tag: append_error_message_30c21dc1
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - set:
@@ -147,8 +153,10 @@ processors:
       if: ctx.okta?.statusChanged != null && ctx.okta.statusChanged != ''
       on_failure:
         - remove:
+            tag: remove_okta_statusChanged_00bd900d
             field: okta.statusChanged
         - append:
+            tag: append_error_message_46c91e09
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - set:
@@ -170,8 +178,10 @@ processors:
       if: ctx.okta?.lastLogin != null && ctx.okta.lastLogin != ''
       on_failure:
         - remove:
+            tag: remove_okta_lastLogin_740fca4e
             field: okta.lastLogin
         - append:
+            tag: append_error_message_b32060bf
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - set:
@@ -193,14 +203,16 @@ processors:
       if: ctx.okta?.lastUpdated != null && ctx.okta.lastUpdated != ''
       on_failure:
         - remove:
+            tag: remove_okta_lastUpdated_d6ba9ca0
             field: okta.lastUpdated
         - append:
+            tag: append_error_message_6bf8bd67
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - set:
       field: asset.last_updated
       copy_from: entityanalytics_okta.user.last_updated
-      tag: set_asset_last_seen
+      tag: set_asset_last_updated_011f75e3
       ignore_empty_value: true
   - date:
       field: okta.passwordChanged
@@ -211,8 +223,10 @@ processors:
       if: ctx.okta?.passwordChanged != null && ctx.okta.passwordChanged != ''
       on_failure:
         - remove:
+            tag: remove_okta_passwordChanged_6bace184
             field: okta.passwordChanged
         - append:
+            tag: append_error_message_0bfc9bbf
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - set:
@@ -231,6 +245,7 @@ processors:
       tag: rename_user_groups
       ignore_missing: true
   - foreach:
+      tag: foreach_entityanalytics_okta_groups_4f84bc86
       field: entityanalytics_okta.groups
       if: ctx.entityanalytics_okta?.groups != null
       processor:
@@ -239,6 +254,7 @@ processors:
           value: '{{{_ingest._value.id}}}'
           allow_duplicates: false
   - foreach:
+      tag: foreach_entityanalytics_okta_groups_43144c3f
       field: entityanalytics_okta.groups
       if: ctx.entityanalytics_okta?.groups != null
       processor:
@@ -252,6 +268,7 @@ processors:
       tag: rename_user_roles
       ignore_missing: true
   - foreach:
+      tag: foreach_entityanalytics_okta_roles_5b1ce602
       field: entityanalytics_okta.roles
       if: ctx.entityanalytics_okta?.roles instanceof List
       processor:
@@ -260,6 +277,7 @@ processors:
           value: '{{{_ingest._value.id}}}'
           allow_duplicates: false
   - foreach:
+      tag: foreach_entityanalytics_okta_roles_bd8589f1
       field: entityanalytics_okta.roles
       if: ctx.entityanalytics_okta?.roles instanceof List
       processor:
@@ -268,6 +286,7 @@ processors:
           value: '{{{_ingest._value.label}}}'
           allow_duplicates: false
   - foreach:
+      tag: foreach_entityanalytics_okta_roles_66618195
       field: entityanalytics_okta.roles
       if: ctx.entityanalytics_okta?.roles instanceof List
       processor:
@@ -277,6 +296,7 @@ processors:
           tag: rename_user_roles.assignmentType
           ignore_missing: true
   - foreach:
+      tag: foreach_entityanalytics_okta_roles_6fb80d55
       field: entityanalytics_okta.roles
       if: ctx.entityanalytics_okta?.roles instanceof List
       processor:
@@ -286,6 +306,7 @@ processors:
           tag: rename_user_roles.assignmentType
           ignore_missing: true
   - foreach:
+      tag: foreach_entityanalytics_okta_roles_6a1b3c29
       field: entityanalytics_okta.roles
       if: ctx.entityanalytics_okta?.roles instanceof List
       processor:
@@ -301,6 +322,7 @@ processors:
                 target_field: _ingest._value.lastUpdated
                 ignore_failure: true
   - foreach:
+      tag: foreach_entityanalytics_okta_roles_1d898d7a
       field: entityanalytics_okta.roles
       if: ctx.entityanalytics_okta?.roles instanceof List
       processor:
@@ -774,10 +796,12 @@ processors:
       allow_duplicates: false
       if: ctx.entityanalytics_okta?.user?.profile?.manager?.name != null
   - set:
+      tag: set_okta_credentials_recovery_question_is_set_a6e69872
       field: okta.credentials.recovery_question.is_set
       value: true
       if: ctx.okta?.credentials?.recovery_question != null
   - set:
+      tag: set_okta_credentials_recovery_question_is_set_12a22165
       field: okta.credentials.recovery_question.is_set
       value: false
       if: ctx.okta?.credentials?.recovery_question == null

packages/entityanalytics_okta/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: entityanalytics_okta
 title: Okta Entity Analytics
-version: "3.4.0"
+version: "3.4.1"
 description: "Collect Identities from Okta with Elastic Agent."
 type: integration
 categories:

packages/eset_protect/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.5.3"
+  changes:
+    - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/20573
 - version: "2.5.2"
   changes:
     - description: Set agentless deployment mode `release` field to `ga`.
@@ -33,7 +38,7 @@
   changes:
     - description: |
         Revert v2 detections endpoint support added in v2.0.0; the detection data stream now uses only the v1 API.
         Users upgrading from v2.0.0 who are using the default API Version (v2) for detection data stream may experience a data gap.
         To recover missing data, create a new policy and configure the Initial Interval setting to cover the duration that v2.0.0 was in use.
       type: breaking-change
       link: https://github.com/elastic/integrations/pull/18177
@@ -124,7 +129,7 @@
       link: https://github.com/elastic/integrations/pull/12697
 - version: "1.5.0"
   changes:
     - description: Add "preserve_original_event" tag to documents with `event.kind` manually set to "pipeline_error".
       type: enhancement
       link: https://github.com/elastic/integrations/pull/12109
 - version: "1.4.0"
@@ -134,7 +139,7 @@
       link: https://github.com/elastic/integrations/pull/12076
 - version: "1.3.0"
   changes:
     - description: Add "preserve_original_event" tag to documents with `event.kind` set to "pipeline_error".
       type: enhancement
       link: https://github.com/elastic/integrations/pull/12046
 - version: "1.2.0"