From 127f8141950fc34e9864f596d842af0edfe12f0c Mon Sep 17 00:00:00 2001 From: kcreddy Date: Thu, 6 Aug 2026 18:38:37 +0530 Subject: [PATCH 1/2] ssi: add processor tags and preserve_original_event on failure (2/4) Tag every ingest pipeline processor across 52 SSI-owned packages (entityanalytics_okta through sentinel_one_cloud_funnel) with a unique, descriptive identifier so that failure telemetry can attribute errors to the specific step that failed rather than collapsing same-type processors into one bucket. Add preserve_original_event to pipeline-level on_failure handlers that were missing it, ensuring the raw payload is retained when a pipeline error document is indexed. Updates #20558 --- packages/entityanalytics_okta/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 4 + .../elasticsearch/ingest_pipeline/device.yml | 17 +- .../elasticsearch/ingest_pipeline/user.yml | 28 +- packages/entityanalytics_okta/manifest.yml | 2 +- packages/eset_protect/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 16 + .../elasticsearch/ingest_pipeline/default.yml | 13 + .../elasticsearch/ingest_pipeline/default.yml | 12 + .../elasticsearch/ingest_pipeline/default.yml | 4 + .../elasticsearch/ingest_pipeline/default.yml | 41 +- packages/eset_protect/manifest.yml | 2 +- packages/first_epss/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 4 + packages/first_epss/manifest.yml | 2 +- packages/forgerock/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 29 + .../elasticsearch/ingest_pipeline/default.yml | 17 + .../elasticsearch/ingest_pipeline/default.yml | 18 + .../elasticsearch/ingest_pipeline/default.yml | 16 + .../elasticsearch/ingest_pipeline/default.yml | 16 + .../elasticsearch/ingest_pipeline/default.yml | 26 + .../elasticsearch/ingest_pipeline/default.yml | 16 + .../elasticsearch/ingest_pipeline/default.yml | 16 + .../elasticsearch/ingest_pipeline/default.yml | 15 + .../elasticsearch/ingest_pipeline/default.yml | 13 + .../elasticsearch/ingest_pipeline/default.yml | 19 + packages/forgerock/manifest.yml | 2 +- packages/gcp/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 294 +++++--- .../elasticsearch/ingest_pipeline/default.yml | 2 + .../elasticsearch/ingest_pipeline/default.yml | 17 + .../elasticsearch/ingest_pipeline/default.yml | 53 +- .../elasticsearch/ingest_pipeline/default.yml | 59 +- .../elasticsearch/ingest_pipeline/default.yml | 37 +- .../elasticsearch/ingest_pipeline/default.yml | 25 + .../elasticsearch/ingest_pipeline/default.yml | 30 +- .../elasticsearch/ingest_pipeline/default.yml | 73 +- .../elasticsearch/ingest_pipeline/default.yml | 8 + .../elasticsearch/ingest_pipeline/default.yml | 88 ++- .../elasticsearch/ingest_pipeline/default.yml | 46 ++ .../elasticsearch/ingest_pipeline/default.yml | 66 +- .../elasticsearch/ingest_pipeline/default.yml | 35 + .../elasticsearch/ingest_pipeline/default.yml | 55 ++ .../elasticsearch/ingest_pipeline/default.yml | 36 + .../elasticsearch/ingest_pipeline/default.yml | 15 + .../elasticsearch/ingest_pipeline/default.yml | 80 +- packages/gcp/manifest.yml | 2 +- packages/github/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 56 +- .../elasticsearch/ingest_pipeline/default.yml | 51 ++ .../elasticsearch/ingest_pipeline/default.yml | 57 ++ .../elasticsearch/ingest_pipeline/default.yml | 31 + .../elasticsearch/ingest_pipeline/default.yml | 57 ++ .../elasticsearch/ingest_pipeline/default.yml | 13 +- packages/github/manifest.yml | 2 +- packages/gitlab/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 120 ++- .../elasticsearch/ingest_pipeline/default.yml | 46 +- .../elasticsearch/ingest_pipeline/default.yml | 32 +- .../elasticsearch/ingest_pipeline/default.yml | 35 +- .../elasticsearch/ingest_pipeline/default.yml | 52 +- .../elasticsearch/ingest_pipeline/default.yml | 79 +- .../elasticsearch/ingest_pipeline/default.yml | 324 +++++++- packages/gitlab/manifest.yml | 2 +- packages/google_scc/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 6 +- .../ingest_pipeline/pipeline_asset.yml | 22 +- .../ingest_pipeline/pipeline_prior_asset.yml | 22 +- .../elasticsearch/ingest_pipeline/default.yml | 42 +- .../elasticsearch/ingest_pipeline/default.yml | 127 +++- .../elasticsearch/ingest_pipeline/default.yml | 4 + packages/google_scc/manifest.yml | 2 +- packages/google_secops/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 58 +- packages/google_secops/manifest.yml | 2 +- packages/google_workspace/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 65 ++ .../elasticsearch/ingest_pipeline/default.yml | 708 +++++------------- .../elasticsearch/ingest_pipeline/default.yml | 195 ++++- .../elasticsearch/ingest_pipeline/default.yml | 29 +- .../elasticsearch/ingest_pipeline/default.yml | 18 +- .../elasticsearch/ingest_pipeline/default.yml | 29 + .../elasticsearch/ingest_pipeline/default.yml | 58 ++ .../elasticsearch/ingest_pipeline/default.yml | 18 +- .../elasticsearch/ingest_pipeline/default.yml | 103 +++ .../elasticsearch/ingest_pipeline/default.yml | 67 +- .../elasticsearch/ingest_pipeline/default.yml | 53 ++ .../elasticsearch/ingest_pipeline/default.yml | 54 +- .../elasticsearch/ingest_pipeline/default.yml | 61 ++ .../elasticsearch/ingest_pipeline/default.yml | 71 +- .../elasticsearch/ingest_pipeline/default.yml | 17 +- .../elasticsearch/ingest_pipeline/default.yml | 59 +- .../elasticsearch/ingest_pipeline/default.yml | 74 +- .../elasticsearch/ingest_pipeline/default.yml | 84 +++ .../elasticsearch/ingest_pipeline/default.yml | 36 + .../elasticsearch/ingest_pipeline/default.yml | 62 ++ .../elasticsearch/ingest_pipeline/default.yml | 34 + .../elasticsearch/ingest_pipeline/default.yml | 20 +- packages/google_workspace/manifest.yml | 2 +- packages/greenhouse/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 32 + packages/greenhouse/manifest.yml | 2 +- packages/ibm_qradar/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 54 +- packages/ibm_qradar/manifest.yml | 2 +- packages/imperva_cloud_waf/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 26 + packages/imperva_cloud_waf/manifest.yml | 2 +- packages/infoblox_bloxone_ddi/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 53 ++ .../elasticsearch/ingest_pipeline/default.yml | 322 ++++++++ .../elasticsearch/ingest_pipeline/default.yml | 101 +++ packages/infoblox_bloxone_ddi/manifest.yml | 2 +- packages/island_browser/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 6 + .../elasticsearch/ingest_pipeline/default.yml | 13 + .../elasticsearch/ingest_pipeline/default.yml | 4 + .../elasticsearch/ingest_pipeline/default.yml | 29 + .../elasticsearch/ingest_pipeline/default.yml | 9 + packages/island_browser/manifest.yml | 2 +- .../jamf_compliance_reporter/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 8 + .../ingest_pipeline/pipeline_app_metrics.yml | 36 + .../pipeline_audio_video_device_event.yml | 12 + .../ingest_pipeline/pipeline_audit.yml | 88 +++ ...ipeline_audit_class_verification_event.yml | 10 + .../ingest_pipeline/pipeline_aue_accept.yml | 10 + .../pipeline_aue_arguments.yml | 6 + .../ingest_pipeline/pipeline_aue_auth.yml | 6 + .../pipeline_aue_bind_and_aue_connect.yml | 16 + .../ingest_pipeline/pipeline_aue_chdir.yml | 17 + .../ingest_pipeline/pipeline_aue_chroot.yml | 18 + .../ingest_pipeline/pipeline_aue_execve.yml | 28 + .../ingest_pipeline/pipeline_aue_exit.yml | 9 + .../ingest_pipeline/pipeline_aue_fork.yml | 8 + .../ingest_pipeline/pipeline_aue_kill.yml | 7 + .../ingest_pipeline/pipeline_aue_listen.yml | 7 + .../ingest_pipeline/pipeline_aue_logout.yml | 6 + .../ingest_pipeline/pipeline_aue_mount.yml | 20 + .../pipeline_aue_pidfortask.yml | 8 + .../pipeline_aue_posix_spawn.yml | 14 + .../ingest_pipeline/pipeline_aue_ptrace.yml | 10 + ...remove_from_group_and_aue_mac_set_proc.yml | 7 + .../ingest_pipeline/pipeline_aue_session.yml | 7 + .../pipeline_aue_setpriority.yml | 9 + .../pipeline_aue_socketpair.yml | 8 + .../pipeline_aue_ssauthint.yml | 8 + .../pipeline_aue_taskforpid.yml | 9 + .../pipeline_aue_tasknameforpid.yml | 9 + .../ingest_pipeline/pipeline_aue_unmount.yml | 18 + .../ingest_pipeline/pipeline_aue_wait4.yml | 7 + ...orter_tamper_event_and_file_event_info.yml | 55 ++ .../ingest_pipeline/pipeline_event.yml | 38 + .../pipeline_exec_chain_child_object.yml | 8 + .../pipeline_gatekeeper_info_event.yml | 10 + .../pipeline_gatekeeper_manual_overrides.yml | 8 + .../pipeline_gatekeeper_quarantine_log.yml | 13 + .../pipeline_hardware_event.yml | 23 + .../pipeline_identity_object.yml | 11 + .../pipeline_license_info_event.yml | 13 + .../pipeline_preference_list_event.yml | 36 + .../pipeline_print_event_information.yml | 18 + .../pipeline_process_object.yml | 32 + .../pipeline_prohibited_app_blocked.yml | 57 ++ .../ingest_pipeline/pipeline_signal_event.yml | 6 + .../pipeline_unified_log_event.yml | 31 + ...line_xprotect_definitions_version_info.yml | 10 + .../pipeline_xprotect_event_log.yml | 37 + .../jamf_compliance_reporter/manifest.yml | 2 +- packages/jamf_pro/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 84 ++- .../elasticsearch/ingest_pipeline/default.yml | 34 +- packages/jamf_pro/manifest.yml | 2 +- packages/jamf_protect/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 120 ++- .../ingest_pipeline/gpunifiedlogevent.yml | 17 +- .../ingest_pipeline/gpusbevent.yml | 23 +- .../elasticsearch/ingest_pipeline/default.yml | 134 +++- .../pipeline_event_authentication.yml | 186 ++++- .../pipeline_event_bios_uefi.yml | 10 +- .../pipeline_event_btm_launch_item_add.yml | 116 ++- .../pipeline_event_btm_launch_item_remove.yml | 118 ++- .../ingest_pipeline/pipeline_event_chroot.yml | 16 +- .../pipeline_event_cs_invalidated.yml | 14 +- .../ingest_pipeline/pipeline_event_exec.yml | 74 +- .../pipeline_event_file_collection.yml | 9 +- ...ipeline_event_gatekeeper_user_override.yml | 29 +- .../pipeline_event_kextload.yml | 19 +- .../pipeline_event_kextunload.yml | 19 +- .../pipeline_event_log_collection.yml | 11 +- .../pipeline_event_login_login.yml | 21 +- .../pipeline_event_login_logout.yml | 20 +- .../pipeline_event_lw_session_lock.yml | 20 +- .../pipeline_event_lw_session_login.yml | 20 +- .../pipeline_event_lw_session_logout.yml | 20 +- .../pipeline_event_lw_session_unlock.yml | 20 +- .../ingest_pipeline/pipeline_event_mount.yml | 30 +- .../pipeline_event_network_connect.yml | 85 ++- .../pipeline_event_od_attribute_set.yml | 76 +- .../pipeline_event_od_attribute_value_add.yml | 74 +- ...peline_event_od_attribute_value_remove.yml | 78 +- .../pipeline_event_od_create_group.yml | 74 +- .../pipeline_event_od_create_user.yml | 76 +- .../pipeline_event_od_delete_group.yml | 76 +- .../pipeline_event_od_delete_user.yml | 76 +- .../pipeline_event_od_disable_user.yml | 76 +- .../pipeline_event_od_enable_user.yml | 76 +- .../pipeline_event_od_group_add.yml | 77 +- .../pipeline_event_od_group_remove.yml | 75 +- .../pipeline_event_od_group_set.yml | 76 +- .../pipeline_event_od_modify_password.yml | 78 +- .../pipeline_event_openssh_login.yml | 27 +- .../pipeline_event_openssh_logout.yml | 27 +- .../pipeline_event_profile_add.yml | 80 +- .../pipeline_event_profile_remove.yml | 79 +- .../pipeline_event_pty_close.yml | 16 +- .../pipeline_event_pty_grant.yml | 16 +- .../pipeline_event_remount.yml | 30 +- .../pipeline_event_screensharing_attach.yml | 32 +- .../pipeline_event_screensharing_detach.yml | 21 +- .../pipeline_event_settime.yml | 18 +- .../ingest_pipeline/pipeline_event_su.yml | 31 +- .../ingest_pipeline/pipeline_event_sudo.yml | 18 +- .../pipeline_event_system_performance.yml | 13 +- .../pipeline_event_tcc_modify.yml | 77 +- .../pipeline_event_unmount.yml | 30 +- .../pipeline_event_xp_malware_detected.yml | 20 +- .../pipeline_event_xp_malware_remediated.yml | 22 +- .../pipeline_instigator_object.yml | 28 +- .../pipeline_object_process.yml | 60 +- .../elasticsearch/ingest_pipeline/default.yml | 17 + .../ingest_pipeline/pipeline_audit.yml | 90 +++ .../ingest_pipeline/pipeline_aue_accept.yml | 6 + .../pipeline_aue_arguments.yml | 2 + .../ingest_pipeline/pipeline_aue_auth.yml | 2 + .../pipeline_aue_bind_and_aue_connect.yml | 12 + .../ingest_pipeline/pipeline_aue_chdir.yml | 13 + .../ingest_pipeline/pipeline_aue_chroot.yml | 14 + .../ingest_pipeline/pipeline_aue_execve.yml | 23 + .../ingest_pipeline/pipeline_aue_exit.yml | 5 + .../ingest_pipeline/pipeline_aue_fork.yml | 4 + .../ingest_pipeline/pipeline_aue_kill.yml | 3 + .../ingest_pipeline/pipeline_aue_listen.yml | 3 + .../ingest_pipeline/pipeline_aue_logout.yml | 2 + .../ingest_pipeline/pipeline_aue_mount.yml | 16 + .../pipeline_aue_pidfortask.yml | 4 + .../pipeline_aue_posix_spawn.yml | 10 + ...remove_from_group_and_aue_mac_set_proc.yml | 3 + .../ingest_pipeline/pipeline_aue_session.yml | 3 + .../pipeline_aue_setpriority.yml | 5 + .../pipeline_aue_socketpair.yml | 4 + .../pipeline_aue_ssauthint.yml | 4 + .../pipeline_aue_taskforpid.yml | 5 + .../pipeline_aue_tasknameforpid.yml | 5 + .../ingest_pipeline/pipeline_aue_unmount.yml | 14 + .../ingest_pipeline/pipeline_event.yml | 27 + .../pipeline_exec_chain_child_object.yml | 4 + .../pipeline_identity_object.yml | 6 + .../pipeline_process_object.yml | 28 + .../pipeline_system_performance_metrics.yml | 40 + .../elasticsearch/ingest_pipeline/default.yml | 142 ++-- .../elasticsearch/ingest_pipeline/default.yml | 148 ++-- packages/jamf_protect/manifest.yml | 2 +- packages/jumpcloud/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 55 ++ packages/jumpcloud/manifest.yml | 2 +- .../changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 31 +- .../manifest.yml | 2 +- packages/keycloak/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 5 + .../elasticsearch/ingest_pipeline/events.yml | 65 +- packages/keycloak/manifest.yml | 2 +- packages/lastpass/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 26 + .../elasticsearch/ingest_pipeline/default.yml | 63 ++ .../elasticsearch/ingest_pipeline/default.yml | 45 ++ packages/lastpass/manifest.yml | 2 +- packages/lumos/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 11 + packages/lumos/manifest.yml | 2 +- packages/lyve_cloud/changelog.yml | 5 + .../ingest_pipeline/audit_lc.yml | 60 +- .../elasticsearch/ingest_pipeline/default.yml | 5 + packages/lyve_cloud/manifest.yml | 2 +- packages/m365_defender/changelog.yml | 5 + .../ingest_pipeline/pipeline_alert.yml | 4 + .../pipeline_app_and_identity.yml | 4 + .../ingest_pipeline/pipeline_device.yml | 4 + .../ingest_pipeline/pipeline_email.yml | 4 + packages/m365_defender/manifest.yml | 2 +- packages/mattermost/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 56 ++ packages/mattermost/manifest.yml | 2 +- packages/menlo/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 41 +- .../elasticsearch/ingest_pipeline/default.yml | 78 +- packages/menlo/manifest.yml | 2 +- .../microsoft_defender_cloud/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 108 ++- .../elasticsearch/ingest_pipeline/default.yml | 75 +- .../microsoft_defender_cloud/manifest.yml | 2 +- .../microsoft_defender_endpoint/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 131 +++- .../elasticsearch/ingest_pipeline/default.yml | 20 +- .../elasticsearch/ingest_pipeline/default.yml | 14 +- .../elasticsearch/ingest_pipeline/default.yml | 15 +- .../microsoft_defender_endpoint/manifest.yml | 2 +- .../changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 70 +- .../manifest.yml | 2 +- packages/microsoft_intune/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 11 + .../elasticsearch/ingest_pipeline/default.yml | 12 + packages/microsoft_intune/manifest.yml | 2 +- packages/microsoft_sentinel/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 18 +- .../elasticsearch/ingest_pipeline/default.yml | 16 + .../elasticsearch/ingest_pipeline/default.yml | 15 + packages/microsoft_sentinel/manifest.yml | 2 +- packages/mimecast/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 29 +- .../elasticsearch/ingest_pipeline/default.yml | 73 +- .../elasticsearch/ingest_pipeline/default.yml | 34 + .../elasticsearch/ingest_pipeline/default.yml | 26 +- .../elasticsearch/ingest_pipeline/default.yml | 23 + .../elasticsearch/ingest_pipeline/default.yml | 7 +- .../ingest_pipeline/v1_pipeline.yml | 75 +- .../ingest_pipeline/v2_pipeline.yml | 78 ++ .../elasticsearch/ingest_pipeline/default.yml | 40 +- .../elasticsearch/ingest_pipeline/default.yml | 32 +- .../elasticsearch/ingest_pipeline/default.yml | 29 + .../elasticsearch/ingest_pipeline/default.yml | 22 + .../elasticsearch/ingest_pipeline/default.yml | 27 + packages/mimecast/manifest.yml | 2 +- packages/miniflux/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 16 +- packages/miniflux/manifest.yml | 2 +- packages/netskope/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 191 +++++ .../elasticsearch/ingest_pipeline/default.yml | 2 + .../ingest_pipeline/pipeline_alerts_v2.yml | 90 ++- .../ingest_pipeline/pipeline_events_v2.yml | 93 ++- .../elasticsearch/ingest_pipeline/default.yml | 272 ++++++- .../elasticsearch/ingest_pipeline/default.yml | 90 ++- packages/netskope/manifest.yml | 2 +- packages/okta/changelog.yml | 5 + .../no_use_flattened_debug.yml | 4 + .../ingest_pipeline/use_flattened_debug.yml | 4 + packages/okta/manifest.yml | 2 +- packages/opencanary/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 28 +- packages/opencanary/manifest.yml | 2 +- packages/panw_cortex_xdr/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 8 + .../ingest_pipeline/v1_pipeline.yml | 104 +++ .../ingest_pipeline/v2_pipeline.yml | 53 +- .../elasticsearch/ingest_pipeline/default.yml | 31 + .../elasticsearch/ingest_pipeline/default.yml | 33 + packages/panw_cortex_xdr/manifest.yml | 2 +- packages/ping_federate/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 18 +- .../elasticsearch/ingest_pipeline/default.yml | 21 +- packages/ping_federate/manifest.yml | 2 +- packages/ping_one/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 67 ++ packages/ping_one/manifest.yml | 2 +- packages/pps/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 19 + packages/pps/manifest.yml | 2 +- packages/prisma_cloud/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 106 ++- .../elasticsearch/ingest_pipeline/default.yml | 18 +- .../elasticsearch/ingest_pipeline/default.yml | 189 ++++- .../elasticsearch/ingest_pipeline/default.yml | 53 +- .../elasticsearch/ingest_pipeline/default.yml | 65 +- .../elasticsearch/ingest_pipeline/default.yml | 12 + .../elasticsearch/ingest_pipeline/default.yml | 19 + packages/prisma_cloud/manifest.yml | 2 +- packages/proofpoint_essentials/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 47 +- packages/proofpoint_essentials/manifest.yml | 2 +- packages/proofpoint_itm/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 12 +- packages/proofpoint_itm/manifest.yml | 2 +- packages/proofpoint_on_demand/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 7 + .../elasticsearch/ingest_pipeline/default.yml | 10 + .../elasticsearch/ingest_pipeline/default.yml | 60 +- packages/proofpoint_on_demand/manifest.yml | 2 +- packages/proofpoint_tap/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 47 ++ .../elasticsearch/ingest_pipeline/default.yml | 47 ++ .../elasticsearch/ingest_pipeline/default.yml | 92 +++ .../elasticsearch/ingest_pipeline/default.yml | 88 +++ packages/proofpoint_tap/manifest.yml | 2 +- packages/qualys_vmdr/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 52 ++ .../pipeline_knowledge_base.yml | 20 +- .../elasticsearch/ingest_pipeline/default.yml | 26 +- .../elasticsearch/ingest_pipeline/default.yml | 29 + packages/qualys_vmdr/manifest.yml | 2 +- packages/rapid7_insightvm/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 94 +++ .../elasticsearch/ingest_pipeline/default.yml | 52 +- .../elasticsearch/ingest_pipeline/default.yml | 74 +- packages/rapid7_insightvm/manifest.yml | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 49 +- packages/santa/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 57 ++ packages/santa/manifest.yml | 2 +- packages/sentinel_one/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 46 +- packages/sentinel_one/manifest.yml | 2 +- .../sentinel_one_cloud_funnel/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 446 +++++++++++ .../pipeline-command-script.yml | 25 + .../pipeline-cross-process.yml | 49 ++ .../ingest_pipeline/pipeline-dns.yml | 17 + .../ingest_pipeline/pipeline-file.yml | 78 ++ .../ingest_pipeline/pipeline-indicator.yml | 18 + .../ingest_pipeline/pipeline-login.yml | 28 + .../ingest_pipeline/pipeline-module.yml | 13 + .../pipeline-network-action.yml | 59 +- .../ingest_pipeline/pipeline-process.yml | 100 ++- .../ingest_pipeline/pipeline-registry.yml | 42 +- .../pipeline-scheduled-task.yml | 36 + ...pipeline-threat-intelligence-indicator.yml | 24 + .../ingest_pipeline/pipeline-url.yml | 14 + .../sentinel_one_cloud_funnel/manifest.yml | 2 +- 431 files changed, 13216 insertions(+), 1717 deletions(-) diff --git a/packages/entityanalytics_okta/changelog.yml b/packages/entityanalytics_okta/changelog.yml index 0c45b6fa392..48e76c8814d 100644 --- a/packages/entityanalytics_okta/changelog.yml +++ b/packages/entityanalytics_okta/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.4.1" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "3.4.0" changes: - description: Add the Advanced Analytics (UEBA) subcategory diff --git a/packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/default.yml b/packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/default.yml index a18ffb49bc2..a3d821d9761 100644 --- a/packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/default.yml +++ b/packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/default.yml @@ -7,6 +7,7 @@ processors: value: 8.11.0 - remove: + tag: remove_event_action_9b3969af field: event.action if: ctx.event?.action != "started" && ctx.event?.action != "completed" ignore_missing: true @@ -24,11 +25,13 @@ processors: tag: pipeline_to_device if: ctx.device?.id != null - rename: + tag: rename_okta_domain_to_host_name_bc4fe597 field: okta_domain target_field: host.name ignore_missing: true on_failure: - rename: + tag: rename_okta_domain_to_okta_okta_domain_79b460fb field: okta_domain target_field: okta.okta_domain ignore_failure: true @@ -57,6 +60,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/device.yml b/packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/device.yml index 1884474a829..6bf5ce113ed 100644 --- a/packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/device.yml +++ b/packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/device.yml @@ -15,6 +15,7 @@ processors: ctx.event.original = stringified_orig; } - remove: + tag: remove_event_action_9b3969af field: event.action if: ctx.event?.action != "started" && ctx.event?.action != "completed" ignore_missing: true @@ -72,8 +73,10 @@ processors: if: ctx.okta?.created != null && ctx.okta.created != '' on_failure: - remove: + tag: remove_okta_created_235a502d field: okta.created - append: + tag: append_error_message_729fae1b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -90,8 +93,10 @@ processors: if: ctx.okta?.activated != null && ctx.okta.activated != '' on_failure: - remove: + tag: remove_okta_activated_064816bc field: okta.activated - append: + tag: append_error_message_66f96d3f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -103,8 +108,10 @@ processors: if: ctx.okta?.statusChanged != null && ctx.okta.statusChanged != '' on_failure: - remove: + tag: remove_okta_statusChanged_01e67889 field: okta.statusChanged - append: + tag: append_error_message_bd1a4715 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -121,8 +128,10 @@ processors: if: ctx.okta?.lastUpdated != null && ctx.okta.lastUpdated != '' on_failure: - remove: + tag: remove_okta_lastUpdated_62fbc0a4 field: okta.lastUpdated - append: + tag: append_error_message_97e5b363 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -207,7 +216,7 @@ processors: - rename: field: okta.profile.diskEncryptionType target_field: entityanalytics_okta.device.profile.disk_encryption_type - tag: rename_device_profile_display_name + tag: rename_okta_profile_diskEncryptionType_to_entityanalytics_okta_device_profile_disk_encryption_type_8161cdc0 ignore_missing: true - convert: field: okta.profile.registered @@ -217,8 +226,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_okta_profile_registered_cd970fe3 field: okta.profile.registered - append: + tag: append_error_message_5aff211e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -229,8 +240,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_okta_profile_secure_hardware_present_a53eaf05 field: okta.profile.secure_hardware_present - append: + tag: append_error_message_703b70b0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -241,8 +254,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_okta_profile_managed_9556352a field: okta.profile.managed - append: + tag: append_error_message_3190ba2a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: diff --git a/packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/user.yml b/packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/user.yml index 566d4c940f1..ae45cd649a7 100644 --- a/packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/user.yml +++ b/packages/entityanalytics_okta/data_stream/entity/elasticsearch/ingest_pipeline/user.yml @@ -15,6 +15,7 @@ processors: ctx.event.original = stringified_orig; } - remove: + tag: remove_event_action_9b3969af field: event.action if: ctx.event?.action != "started" && ctx.event?.action != "completed" ignore_missing: true @@ -29,7 +30,7 @@ processors: - set: field: event.type tag: set_event_type - value: ['user','info'] + value: ['user', 'info'] - set: field: asset.category tag: set_asset_category @@ -80,6 +81,7 @@ processors: } on_failure: - append: + tag: append_error_message_1c58f509 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -106,8 +108,10 @@ processors: if: ctx.okta?.created != null && ctx.okta.created != '' on_failure: - remove: + tag: remove_okta_created_400119ef field: okta.created - append: + tag: append_error_message_5d1ccd5d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -129,8 +133,10 @@ processors: if: ctx.okta?.activated != null && ctx.okta.activated != '' on_failure: - remove: + tag: remove_okta_activated_69d3eb42 field: okta.activated - append: + tag: append_error_message_30c21dc1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -147,8 +153,10 @@ processors: if: ctx.okta?.statusChanged != null && ctx.okta.statusChanged != '' on_failure: - remove: + tag: remove_okta_statusChanged_00bd900d field: okta.statusChanged - append: + tag: append_error_message_46c91e09 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -170,8 +178,10 @@ processors: if: ctx.okta?.lastLogin != null && ctx.okta.lastLogin != '' on_failure: - remove: + tag: remove_okta_lastLogin_740fca4e field: okta.lastLogin - append: + tag: append_error_message_b32060bf field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -193,14 +203,16 @@ processors: if: ctx.okta?.lastUpdated != null && ctx.okta.lastUpdated != '' on_failure: - remove: + tag: remove_okta_lastUpdated_d6ba9ca0 field: okta.lastUpdated - append: + tag: append_error_message_6bf8bd67 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: asset.last_updated copy_from: entityanalytics_okta.user.last_updated - tag: set_asset_last_seen + tag: set_asset_last_updated_011f75e3 ignore_empty_value: true - date: field: okta.passwordChanged @@ -211,8 +223,10 @@ processors: if: ctx.okta?.passwordChanged != null && ctx.okta.passwordChanged != '' on_failure: - remove: + tag: remove_okta_passwordChanged_6bace184 field: okta.passwordChanged - append: + tag: append_error_message_0bfc9bbf field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -231,6 +245,7 @@ processors: tag: rename_user_groups ignore_missing: true - foreach: + tag: foreach_entityanalytics_okta_groups_4f84bc86 field: entityanalytics_okta.groups if: ctx.entityanalytics_okta?.groups != null processor: @@ -239,6 +254,7 @@ processors: value: '{{{_ingest._value.id}}}' allow_duplicates: false - foreach: + tag: foreach_entityanalytics_okta_groups_43144c3f field: entityanalytics_okta.groups if: ctx.entityanalytics_okta?.groups != null processor: @@ -252,6 +268,7 @@ processors: tag: rename_user_roles ignore_missing: true - foreach: + tag: foreach_entityanalytics_okta_roles_5b1ce602 field: entityanalytics_okta.roles if: ctx.entityanalytics_okta?.roles instanceof List processor: @@ -260,6 +277,7 @@ processors: value: '{{{_ingest._value.id}}}' allow_duplicates: false - foreach: + tag: foreach_entityanalytics_okta_roles_bd8589f1 field: entityanalytics_okta.roles if: ctx.entityanalytics_okta?.roles instanceof List processor: @@ -268,6 +286,7 @@ processors: value: '{{{_ingest._value.label}}}' allow_duplicates: false - foreach: + tag: foreach_entityanalytics_okta_roles_66618195 field: entityanalytics_okta.roles if: ctx.entityanalytics_okta?.roles instanceof List processor: @@ -277,6 +296,7 @@ processors: tag: rename_user_roles.assignmentType ignore_missing: true - foreach: + tag: foreach_entityanalytics_okta_roles_6fb80d55 field: entityanalytics_okta.roles if: ctx.entityanalytics_okta?.roles instanceof List processor: @@ -286,6 +306,7 @@ processors: tag: rename_user_roles.assignmentType ignore_missing: true - foreach: + tag: foreach_entityanalytics_okta_roles_6a1b3c29 field: entityanalytics_okta.roles if: ctx.entityanalytics_okta?.roles instanceof List processor: @@ -301,6 +322,7 @@ processors: target_field: _ingest._value.lastUpdated ignore_failure: true - foreach: + tag: foreach_entityanalytics_okta_roles_1d898d7a field: entityanalytics_okta.roles if: ctx.entityanalytics_okta?.roles instanceof List processor: @@ -774,10 +796,12 @@ processors: allow_duplicates: false if: ctx.entityanalytics_okta?.user?.profile?.manager?.name != null - set: + tag: set_okta_credentials_recovery_question_is_set_a6e69872 field: okta.credentials.recovery_question.is_set value: true if: ctx.okta?.credentials?.recovery_question != null - set: + tag: set_okta_credentials_recovery_question_is_set_12a22165 field: okta.credentials.recovery_question.is_set value: false if: ctx.okta?.credentials?.recovery_question == null diff --git a/packages/entityanalytics_okta/manifest.yml b/packages/entityanalytics_okta/manifest.yml index 3f6f21d6113..38aeba84536 100644 --- a/packages/entityanalytics_okta/manifest.yml +++ b/packages/entityanalytics_okta/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: entityanalytics_okta title: Okta Entity Analytics -version: "3.4.0" +version: "3.4.1" description: "Collect Identities from Okta with Elastic Agent." type: integration categories: diff --git a/packages/eset_protect/changelog.yml b/packages/eset_protect/changelog.yml index 463e0f4bd5d..dd926d42ec1 100644 --- a/packages/eset_protect/changelog.yml +++ b/packages/eset_protect/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.5.3" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "2.5.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/eset_protect/data_stream/detection/elasticsearch/ingest_pipeline/default.yml b/packages/eset_protect/data_stream/detection/elasticsearch/ingest_pipeline/default.yml index 9885e366016..b544030a1a3 100644 --- a/packages/eset_protect/data_stream/detection/elasticsearch/ingest_pipeline/default.yml +++ b/packages/eset_protect/data_stream/detection/elasticsearch/ingest_pipeline/default.yml @@ -28,6 +28,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - fingerprint: @@ -147,6 +148,7 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_json_deviceInfo_primaryIp_fa012743 field: json.deviceInfo.primaryIp ignore_missing: true - append: @@ -169,6 +171,7 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_json_deviceInfo_publicIp_783fc47c field: json.deviceInfo.publicIp ignore_missing: true - append: @@ -264,6 +267,7 @@ processors: if: ctx.eset_protect?.detection?.context?.process?.path != '' on_failure: - append: + tag: append_error_message_b5f5c8ad field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - rename: @@ -283,6 +287,7 @@ processors: if: ctx.eset_protect?.detection?.context?.user_name != '' on_failure: - append: + tag: append_error_message_98c4956d field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - append: @@ -320,6 +325,7 @@ processors: if: ctx.json?.networkCommunication?.localIpAddress != null && ctx.json.networkCommunication.localIpAddress != '' on_failure: - append: + tag: append_error_message_0747180d field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - set: @@ -365,6 +371,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e2967448 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - set: @@ -396,6 +403,7 @@ processors: if: ctx.json?.networkCommunication?.remoteIpAddress != null && ctx.json.networkCommunication.remoteIpAddress != '' on_failure: - append: + tag: append_error_message_4126186a field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - set: @@ -441,6 +449,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_054d9ba5 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - set: @@ -503,6 +512,7 @@ processors: if: ctx.json?.occurTime != null && ctx.json.occurTime != '' on_failure: - append: + tag: append_error_message_f3654959 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - set: @@ -511,6 +521,7 @@ processors: copy_from: eset_protect.detection.occur_time ignore_empty_value: true - foreach: + tag: foreach_json_responses_382d0f1a field: json.responses if: ctx.json?.responses instanceof List processor: @@ -528,6 +539,7 @@ processors: field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - foreach: + tag: foreach_json_responses_2139ba6b field: json.responses if: ctx.json?.responses instanceof List processor: @@ -537,6 +549,7 @@ processors: target_field: _ingest._value.display_name ignore_missing: true - foreach: + tag: foreach_json_responses_888ddb2c field: json.responses if: ctx.json?.responses instanceof List processor: @@ -546,6 +559,7 @@ processors: target_field: _ingest._value.protection_name ignore_missing: true - foreach: + tag: foreach_json_responses_22da6fd4 field: json.responses if: ctx.json?.responses instanceof List processor: @@ -617,11 +631,13 @@ processors: } drop(ctx); - append: + tag: append_event_kind_d2f25e35 field: event.kind value: pipeline_error allow_duplicates: false if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/eset_protect/data_stream/device/elasticsearch/ingest_pipeline/default.yml b/packages/eset_protect/data_stream/device/elasticsearch/ingest_pipeline/default.yml index 07b2fc9ec32..f5ffff9df48 100644 --- a/packages/eset_protect/data_stream/device/elasticsearch/ingest_pipeline/default.yml +++ b/packages/eset_protect/data_stream/device/elasticsearch/ingest_pipeline/default.yml @@ -28,6 +28,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - set: @@ -81,9 +82,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_json_primaryLocalIpAddress_621acdb6 field: json.primaryLocalIpAddress ignore_missing: true - append: + tag: append_error_message_239ac3d1 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - append: @@ -106,9 +109,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_json_publicIpAddress_8ce77ce7 field: json.publicIpAddress ignore_missing: true - append: + tag: append_error_message_25ec2e56 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - append: @@ -208,6 +213,7 @@ processors: } on_failure: - append: + tag: append_error_message_b9d8dd63 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - script: @@ -246,6 +252,7 @@ processors: } on_failure: - append: + tag: append_error_message_7c0ce348 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - fingerprint: @@ -315,6 +322,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2e4b68d6 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - rename: @@ -335,6 +343,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ce87b32d field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - convert: @@ -345,6 +354,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_36bb32fc field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - convert: @@ -355,6 +365,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_abeea48a field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - rename: @@ -421,11 +432,13 @@ processors: } drop(ctx); - append: + tag: append_event_kind_d2f25e35 field: event.kind value: pipeline_error allow_duplicates: false if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/eset_protect/data_stream/device_task/elasticsearch/ingest_pipeline/default.yml b/packages/eset_protect/data_stream/device_task/elasticsearch/ingest_pipeline/default.yml index ff4ff92dd0b..9e444b86259 100644 --- a/packages/eset_protect/data_stream/device_task/elasticsearch/ingest_pipeline/default.yml +++ b/packages/eset_protect/data_stream/device_task/elasticsearch/ingest_pipeline/default.yml @@ -28,6 +28,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - set: @@ -56,6 +57,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4e63ca9d field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - convert: @@ -66,6 +68,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_3162e0d9 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - rename: @@ -80,9 +83,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_json_action_params_restart_8fee63d0 field: json.action.params.restart ignore_missing: true - append: + tag: append_error_message_eea69420 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - rename: @@ -122,6 +127,7 @@ processors: target_field: eset_protect.device_task.targets.device_groups_uuids ignore_missing: true - foreach: + tag: foreach_json_targets_devicesUuids_9d313664 field: json.targets.devicesUuids if: ctx.json?.targets?.devicesUuids instanceof List processor: @@ -136,6 +142,7 @@ processors: target_field: eset_protect.device_task.targets.devices_uuids ignore_missing: true - foreach: + tag: foreach_eset_protect_device_task_targets_devices_uuids_abd03c3f field: eset_protect.device_task.targets.devices_uuids if: ctx.eset_protect?.device_task?.targets?.devices_uuids instanceof List processor: @@ -145,6 +152,7 @@ processors: value: "{{{_ingest._value}}}" allow_duplicates: false - foreach: + tag: foreach_json_triggers_504c9a2b field: json.triggers if: ctx.json?.triggers instanceof List processor: @@ -159,6 +167,7 @@ processors: field: _ingest._value.manual.createTime ignore_missing: true - foreach: + tag: foreach_json_triggers_243173bf field: json.triggers if: ctx.json?.triggers instanceof List processor: @@ -173,6 +182,7 @@ processors: field: _ingest._value.manual.expireTime ignore_missing: true - foreach: + tag: foreach_json_triggers_6fdb63fe field: json.triggers if: ctx.json?.triggers instanceof List processor: @@ -228,11 +238,13 @@ processors: } drop(ctx); - append: + tag: append_event_kind_d2f25e35 field: event.kind value: pipeline_error allow_duplicates: false if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/eset_protect/data_stream/device_vulnerability/elasticsearch/ingest_pipeline/default.yml b/packages/eset_protect/data_stream/device_vulnerability/elasticsearch/ingest_pipeline/default.yml index 8c35ba97b2b..8d71db32695 100644 --- a/packages/eset_protect/data_stream/device_vulnerability/elasticsearch/ingest_pipeline/default.yml +++ b/packages/eset_protect/data_stream/device_vulnerability/elasticsearch/ingest_pipeline/default.yml @@ -28,6 +28,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - set: @@ -190,6 +191,7 @@ processors: ctx.json.remove('packageVulnerability'); on_failure: - append: + tag: append_error_message_5b02ac56 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - rename: @@ -250,11 +252,13 @@ processors: } drop(ctx); - append: + tag: append_event_kind_d2f25e35 field: event.kind value: pipeline_error allow_duplicates: false if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/eset_protect/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/eset_protect/data_stream/event/elasticsearch/ingest_pipeline/default.yml index d1c651f8e81..d7ed0aa883c 100644 --- a/packages/eset_protect/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/eset_protect/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -37,16 +37,14 @@ processors: if: "ctx.event?.original.startsWith(\"{\\\"\")" on_failure: - append: + tag: append_error_message_d91c6690 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - grok: field: event.original tag: grok_event_original patterns: - - 'Detection type: %{DATA:json.threat_name} Detection name: %{DATA:json.threat_name} - Computer name: %{HOSTNAME:json.hostname} Logged user: %{DATA:user.domain}\\%{DATA:json.user} - Time of occurrence: %{ESET_TIMESTAMP:json.occurred_plaintext} Scanner: %{DATA:json.scanner_id} - Action performed: %{DATA:json.action_taken}$' + - 'Detection type: %{DATA:json.threat_name} Detection name: %{DATA:json.threat_name} Computer name: %{HOSTNAME:json.hostname} Logged user: %{DATA:user.domain}\\%{DATA:json.user} Time of occurrence: %{ESET_TIMESTAMP:json.occurred_plaintext} Scanner: %{DATA:json.scanner_id} Action performed: %{DATA:json.action_taken}$' - '^%{DATA:message}\s+%{HOSTNAME:json.hostname} %{ESET_TIMESTAMP:json.occurred_plaintext} %{DATA:eset_protect.event.threat_type} %{NOTSPACE:threat.indicator.name} %{ESET_THREAT_INDICATOR_TYPE:json.object_type} %{GREEDYDATA:json.object_uri}$' pattern_definitions: ESET_TIMESTAMP: "%{MONTHNUM}/%{MONTHDAY}/%{YEAR}, %{HOUR}:%{MINUTE}:%{SECOND} (?:AM|PM) UTC\\+%{INT}" @@ -54,9 +52,11 @@ processors: if: "! ctx.event?.original.startsWith(\"{\\\"\")" on_failure: - append: + tag: append_error_message_a32a7c6e field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - append: + tag: append_tags_deff0e6f field: tags value: eset_notification allow_duplicates: false @@ -131,7 +131,7 @@ processors: tag: grok_host_hostname_and_domain_from_event_hostname patterns: - '(%{DATA:host.hostname}\.%{GREEDYDATA:host.domain}|%{GREEDYDATA:host.hostname})' - if: ctx.eset_protect?.event?.hostname != null + if: ctx.eset_protect?.event?.hostname != null - set: field: host.name tag: set_host_name_from_host_hostname @@ -162,6 +162,7 @@ processors: if: ctx.json?.ipv4 != null && ctx.json.ipv4 != '' on_failure: - append: + tag: append_error_message_5293279b field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - append: @@ -185,6 +186,7 @@ processors: if: ctx.json?.ipv6 != null && ctx.json.ipv6 != '' on_failure: - append: + tag: append_error_message_41a9c475 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - append: @@ -207,6 +209,7 @@ processors: description: Fallback for legacy events with the misspelled "occured" field. Renamed to correct spelling "occurred". if: ctx.json?.occurred == null - gsub: + tag: gsub_json_occurred_plaintext_to_json_occurred_28de6d8e field: json.occurred_plaintext pattern: UTC([+-])([0-9])$ replacement: UTC$10$2:00 @@ -222,6 +225,7 @@ processors: if: ctx.json?.occurred != null && ctx.json.occurred != '' on_failure: - append: + tag: append_error_message_668ce04e field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - set: @@ -320,6 +324,7 @@ processors: if: ctx.event?.action != '' on_failure: - append: + tag: append_error_message_ce1e3af4 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - join: @@ -329,6 +334,7 @@ processors: if: ctx.event?.action instanceof List on_failure: - append: + tag: append_error_message_b7327b72 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - rename: @@ -344,6 +350,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d7386810 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - set: @@ -384,6 +391,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_0dafe7d1 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - convert: @@ -394,6 +402,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_aa16f3d0 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - rename: @@ -466,6 +475,7 @@ processors: if: ctx.json?.firstseen != null && ctx.json.firstseen != '' on_failure: - append: + tag: append_error_message_3f59662c field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - set: @@ -518,9 +528,9 @@ processors: if: ctx.eset_protect?.event?.hash != null - lowercase: field: related.hash - tag: lowercase_file_hash_sha1 + tag: lowercase_related_hash_030be11b ignore_missing: true - - convert: + - convert: field: json.inbound tag: convert_inbound_to_boolean target_field: eset_protect.event.inbound @@ -528,6 +538,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_1f10d3b2 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - set: @@ -543,6 +554,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4794abd3 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - rename: @@ -550,7 +562,7 @@ processors: tag: rename_object_type target_field: eset_protect.event.object_type ignore_missing: true - - set: + - set: field: file.type value: file tag: set_file_type_file @@ -617,13 +629,16 @@ processors: } on_failure: - append: + tag: append_error_message_eed8612c field: error.message value: "object_uri for mails could not be parsed: {{{eset_protect.event.object_uri}}}" - append: + tag: append_event_kind_e21d150f field: event.kind value: pipeline_error allow_duplicates: false - set: + tag: set_file_path_7a202c69 field: file.path if: "ctx.eset_protect?.event?.object_uri != null && ctx.eset_protect?.event?.object_type?.toLowerCase() == 'file'" copy_from: eset_protect.event.object_uri @@ -672,6 +687,7 @@ processors: if: ctx.eset_protect?.event?.processname != '' on_failure: - append: + tag: append_error_message_7aa19fd0 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - rename: @@ -748,6 +764,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a0e61097 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - set: @@ -764,6 +781,7 @@ processors: if: ctx.json?.source_address != null && ctx.json.source_address != '' on_failure: - append: + tag: append_error_message_7e6fdab6 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - set: @@ -814,6 +832,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_18b0988a field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - set: @@ -835,6 +854,7 @@ processors: if: ctx.json?.target_address != null && ctx.json.target_address != '' on_failure: - append: + tag: append_error_message_09902cae field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - set: @@ -885,6 +905,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_fb0a9392 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - set: @@ -905,6 +926,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_743555f4 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - rename: @@ -944,6 +966,7 @@ processors: if: ctx.eset_protect?.event?.username != '' on_failure: - append: + tag: append_error_message_d8f21098 field: error.message value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}" - append: @@ -1000,11 +1023,13 @@ processors: } drop(ctx); - append: + tag: append_event_kind_d2f25e35 field: event.kind value: pipeline_error allow_duplicates: false if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/eset_protect/manifest.yml b/packages/eset_protect/manifest.yml index 0bd6b83d190..5ea0d062511 100644 --- a/packages/eset_protect/manifest.yml +++ b/packages/eset_protect/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: eset_protect title: ESET PROTECT -version: "2.5.2" +version: "2.5.3" description: Collect logs from ESET PROTECT with Elastic Agent. type: integration categories: diff --git a/packages/first_epss/changelog.yml b/packages/first_epss/changelog.yml index 3719b83721f..acbb9fd288f 100644 --- a/packages/first_epss/changelog.yml +++ b/packages/first_epss/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.4.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.4.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/first_epss/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml b/packages/first_epss/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml index aad0fb453df..c79d7022a34 100644 --- a/packages/first_epss/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml +++ b/packages/first_epss/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml @@ -24,6 +24,7 @@ processors: value: info allow_duplicates: false - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -41,6 +42,7 @@ processors: if: ctx.event?.original instanceof String on_failure: - append: + tag: append_error_message_b2386695 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -65,6 +67,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d2ec3863 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -75,6 +78,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_1109378b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: diff --git a/packages/first_epss/manifest.yml b/packages/first_epss/manifest.yml index 70c748bce11..d8bdc70ce82 100644 --- a/packages/first_epss/manifest.yml +++ b/packages/first_epss/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: first_epss title: First EPSS -version: "1.4.1" +version: "1.4.2" description: Collect exploit prediction score data from the First EPSS API with Elastic Agent. type: integration categories: diff --git a/packages/forgerock/changelog.yml b/packages/forgerock/changelog.yml index 3716a8898ef..178187ddc39 100644 --- a/packages/forgerock/changelog.yml +++ b/packages/forgerock/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.24.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.24.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/forgerock/data_stream/am_access/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/am_access/elasticsearch/ingest_pipeline/default.yml index dcc2223e6dc..38917dc11d9 100644 --- a/packages/forgerock/data_stream/am_access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/am_access/elasticsearch/ingest_pipeline/default.yml @@ -2,12 +2,15 @@ description: Pipeline for ForgeRock am-access audit logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - set: + tag: set_observer_vendor_a4252fc1 field: observer.vendor value: ForgeRock Identity Platform - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -20,20 +23,25 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - rename: + tag: rename_json_payload_to_forgerock_12e49639 field: json.payload target_field: forgerock - set: + tag: set_event_id_61b21b41 field: event.id copy_from: forgerock._id ignore_failure: true - set: + tag: set_transaction_id_04cf8970 field: transaction.id copy_from: forgerock.transactionId ignore_failure: true - date: + tag: date_forgerock_timestamp_to_timestamp_26758e33 field: forgerock.timestamp formats: - ISO8601 @@ -41,74 +49,91 @@ processors: timezone: UTC on_failure: - append: + tag: append_error_message_e1edf820 field: error.message value: '{{{ _ingest.on_failure_message }}}' # am-access processing - set: + tag: set_event_type_24c7243c field: event.type value: [access] - set: + tag: set_event_action_03f124af field: event.action copy_from: forgerock.eventName ignore_failure: true - set: + tag: set_user_id_374bda8c field: user.id copy_from: forgerock.userId ignore_failure: true - set: + tag: set_client_domain_8e313e84 field: client.domain copy_from: forgerock.client.host ignore_failure: true - convert: + tag: convert_forgerock_client_ip_to_client_ip_16f6010c field: forgerock.client.ip target_field: client.ip type: ip ignore_missing: true - set: + tag: set_client_port_43205056 field: client.port copy_from: forgerock.client.port ignore_failure: true - set: + tag: set_server_domain_654f70fc field: server.domain copy_from: forgerock.server.host ignore_failure: true - convert: + tag: convert_forgerock_server_ip_to_server_ip_b8435e6c field: forgerock.server.ip target_field: server.ip type: ip ignore_missing: true - set: + tag: set_http_request_method_023d8f1e field: http.request.method copy_from: forgerock.http.request.method ignore_failure: true - set: + tag: set_http_request_Path_d840b8d2 field: http.request.Path copy_from: forgerock.http.request.path ignore_failure: true - convert: + tag: convert_forgerock_response_statusCode_to_http_response_status_code_f0bf401c target_field: http.response.status_code field: forgerock.response.statusCode type: integer ignore_failure: true - set: + tag: set_event_outcome_9d188049 if: ctx.forgerock?.response?.status == "SUCCESSFUL" field: event.outcome value: "success" ignore_failure: true - set: + tag: set_event_outcome_88b951f3 if: ctx.forgerock?.response?.status == "FAILED" field: event.outcome value: "failure" ignore_failure: true - convert: + tag: convert_forgerock_response_elapsedTime_1a480219 field: forgerock.response.elapsedTime type: long ignore_failure: true - set: + tag: set_event_duration_769e42e1 field: event.duration copy_from: forgerock.response.elapsedTime ignore_empty_value: true - script: + tag: script_37c8fd18 lang: painless if: ctx.event?.duration != null && ctx.forgerock?.response?.elapsedTimeUnits == 'MILLISECONDS' params: @@ -116,16 +141,19 @@ processors: source: >- ctx.event.duration *= params.MS_TO_NS; - set: + tag: set_service_name_d24027af field: service.name copy_from: forgerock.component ignore_failure: true # cleaning up - remove: + tag: remove_ca120718 field: - json.timestamp - json.type ignore_failure: true - remove: + tag: remove_a9b96086 if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - json @@ -147,6 +175,7 @@ processors: ignore_failure: true ignore_missing: true - script: + tag: script_9d6fa8ba description: Drops null/empty values recursively lang: painless ignore_failure: true diff --git a/packages/forgerock/data_stream/am_activity/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/am_activity/elasticsearch/ingest_pipeline/default.yml index b4541d9455c..b7304a368de 100644 --- a/packages/forgerock/data_stream/am_activity/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/am_activity/elasticsearch/ingest_pipeline/default.yml @@ -2,12 +2,15 @@ description: Pipeline for ForgeRock am-activity audit logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - set: + tag: set_observer_vendor_a4252fc1 field: observer.vendor value: ForgeRock Identity Platform - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -20,20 +23,25 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - rename: + tag: rename_json_payload_to_forgerock_12e49639 field: json.payload target_field: forgerock - set: + tag: set_event_id_61b21b41 field: event.id copy_from: forgerock._id ignore_failure: true - set: + tag: set_transaction_id_04cf8970 field: transaction.id copy_from: forgerock.transactionId ignore_failure: true - date: + tag: date_forgerock_timestamp_to_timestamp_26758e33 field: forgerock.timestamp formats: - ISO8601 @@ -41,36 +49,44 @@ processors: timezone: UTC on_failure: - append: + tag: append_error_message_e1edf820 field: error.message value: '{{{ _ingest.on_failure_message }}}' # parse am-activity - set: + tag: set_event_action_03f124af field: event.action copy_from: forgerock.eventName ignore_failure: true - set: + tag: set_service_name_d24027af field: service.name copy_from: forgerock.component ignore_failure: true - set: + tag: set_user_id_374bda8c field: user.id copy_from: forgerock.userId ignore_failure: true - set: + tag: set_user_effective_id_9b10f540 field: user.effective.id copy_from: forgerock.runAs ignore_failure: true - set: + tag: set_event_reason_23324b63 field: event.reason copy_from: forgerock.operation ignore_failure: true # cleaning up - remove: + tag: remove_ca120718 field: - json.timestamp - json.type ignore_failure: true - remove: + tag: remove_9853f29a if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - json @@ -85,6 +101,7 @@ processors: ignore_failure: true ignore_missing: true - script: + tag: script_9d6fa8ba description: Drops null/empty values recursively lang: painless ignore_failure: true diff --git a/packages/forgerock/data_stream/am_authentication/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/am_authentication/elasticsearch/ingest_pipeline/default.yml index 6ea1dbf22ea..fa5ab0b8783 100644 --- a/packages/forgerock/data_stream/am_authentication/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/am_authentication/elasticsearch/ingest_pipeline/default.yml @@ -2,12 +2,15 @@ description: Pipeline for ForgeRock am-authentication audit logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - set: + tag: set_observer_vendor_a4252fc1 field: observer.vendor value: ForgeRock Identity Platform - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -20,20 +23,25 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - rename: + tag: rename_json_payload_to_forgerock_12e49639 field: json.payload target_field: forgerock - set: + tag: set_event_id_61b21b41 field: event.id copy_from: forgerock._id ignore_failure: true - set: + tag: set_transaction_id_04cf8970 field: transaction.id copy_from: forgerock.transactionId ignore_failure: true - date: + tag: date_forgerock_timestamp_to_timestamp_26758e33 field: forgerock.timestamp formats: - ISO8601 @@ -41,41 +49,50 @@ processors: timezone: UTC on_failure: - append: + tag: append_error_message_e1edf820 field: error.message value: '{{{ _ingest.on_failure_message }}}' # am-authentication processing - set: + tag: set_event_category_e43d8992 field: event.category value: [authentication] - set: + tag: set_event_action_03f124af field: event.action copy_from: forgerock.eventName ignore_failure: true - set: + tag: set_event_outcome_9c5049a6 if: ctx.forgerock?.result == "SUCCESSFUL" field: event.outcome value: "success" ignore_failure: true - set: + tag: set_event_outcome_22d6391d if: ctx?.forgerock?.result == "FAILED" field: event.outcome value: "failure" ignore_failure: true - set: + tag: set_user_id_374bda8c field: user.id copy_from: forgerock.userId ignore_failure: true - set: + tag: set_service_name_d24027af field: service.name copy_from: forgerock.component ignore_failure: true # cleaning up - remove: + tag: remove_ca120718 field: - json.timestamp - json.type ignore_failure: true - remove: + tag: remove_9b381ff5 if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - json @@ -89,6 +106,7 @@ processors: ignore_failure: true ignore_missing: true - script: + tag: script_9d6fa8ba description: Drops null/empty values recursively lang: painless ignore_failure: true diff --git a/packages/forgerock/data_stream/am_config/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/am_config/elasticsearch/ingest_pipeline/default.yml index c3b44daea94..a5699e47245 100644 --- a/packages/forgerock/data_stream/am_config/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/am_config/elasticsearch/ingest_pipeline/default.yml @@ -2,12 +2,15 @@ description: Pipeline for ForgeRock am-confg audit logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - set: + tag: set_observer_vendor_a4252fc1 field: observer.vendor value: ForgeRock Identity Platform - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -20,20 +23,25 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - rename: + tag: rename_json_payload_to_forgerock_12e49639 field: json.payload target_field: forgerock - set: + tag: set_event_id_61b21b41 field: event.id copy_from: forgerock._id ignore_failure: true - set: + tag: set_transaction_id_04cf8970 field: transaction.id copy_from: forgerock.transactionId ignore_failure: true - date: + tag: date_forgerock_timestamp_to_timestamp_26758e33 field: forgerock.timestamp formats: - ISO8601 @@ -41,31 +49,38 @@ processors: timezone: UTC on_failure: - append: + tag: append_error_message_e1edf820 field: error.message value: '{{{ _ingest.on_failure_message }}}' # am-config processing - set: + tag: set_event_category_e56470ca field: event.category value: [configuration] - set: + tag: set_event_action_03f124af field: event.action copy_from: forgerock.eventName ignore_failure: true - set: + tag: set_user_id_374bda8c field: user.id copy_from: forgerock.userId ignore_failure: true - set: + tag: set_user_effective_id_9b10f540 field: user.effective.id copy_from: forgerock.runAs ignore_failure: true # cleaning up - remove: + tag: remove_ca120718 field: - json.timestamp - json.type ignore_failure: true - remove: + tag: remove_a017d454 if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - json @@ -78,6 +93,7 @@ processors: ignore_failure: true ignore_missing: true - script: + tag: script_9d6fa8ba description: Drops null/empty values recursively lang: painless ignore_failure: true diff --git a/packages/forgerock/data_stream/am_core/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/am_core/elasticsearch/ingest_pipeline/default.yml index abcd65a0c64..26d8ab65787 100644 --- a/packages/forgerock/data_stream/am_core/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/am_core/elasticsearch/ingest_pipeline/default.yml @@ -2,12 +2,15 @@ description: Pipeline for ForgeRock am-core debug logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - set: + tag: set_observer_vendor_a4252fc1 field: observer.vendor value: ForgeRock Identity Platform - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -20,16 +23,20 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - rename: + tag: rename_json_payload_to_forgerock_12e49639 field: json.payload target_field: forgerock - set: + tag: set_transaction_id_04cf8970 field: transaction.id copy_from: forgerock.transactionId ignore_failure: true - date: + tag: date_forgerock_timestamp_to_timestamp_26758e33 field: forgerock.timestamp formats: - ISO8601 @@ -37,37 +44,45 @@ processors: timezone: UTC on_failure: - append: + tag: append_error_message_e1edf820 field: error.message value: '{{{ _ingest.on_failure_message }}}' # am-core processing - set: + tag: set_event_reason_b2a64dc5 field: event.reason copy_from: forgerock.message ignore_failure: true - set: + tag: set_log_level_3c18de8e field: log.level copy_from: forgerock.level ignore_failure: true - set: + tag: set_log_logger_053ab4be field: log.logger copy_from: forgerock.logger ignore_failure: true - set: + tag: set_process_name_ce2aa2d0 field: process.name copy_from: forgerock.thread ignore_failure: true - set: + tag: set_error_stack_trace_f1057393 field: error.stack_trace copy_from: forgerock.exception ignore_failure: true # cleaning up - remove: + tag: remove_dc3ac849 field: - json.timestamp - json.type - forgerock.mdc.transactionId ignore_failure: true - remove: + tag: remove_410de790 if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - json @@ -81,6 +96,7 @@ processors: ignore_failure: true ignore_missing: true - script: + tag: script_9d6fa8ba description: Drops null/empty values recursively lang: painless ignore_failure: true diff --git a/packages/forgerock/data_stream/idm_access/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/idm_access/elasticsearch/ingest_pipeline/default.yml index 2549c4cff97..c68614e9bc0 100644 --- a/packages/forgerock/data_stream/idm_access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/idm_access/elasticsearch/ingest_pipeline/default.yml @@ -2,12 +2,15 @@ description: Pipeline for ForgeRock idm-access audit logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - set: + tag: set_observer_vendor_a4252fc1 field: observer.vendor value: ForgeRock Identity Platform - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -20,20 +23,25 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - rename: + tag: rename_json_payload_to_forgerock_12e49639 field: json.payload target_field: forgerock - set: + tag: set_event_id_61b21b41 field: event.id copy_from: forgerock._id ignore_failure: true - set: + tag: set_transaction_id_04cf8970 field: transaction.id copy_from: forgerock.transactionId ignore_failure: true - date: + tag: date_forgerock_timestamp_to_timestamp_26758e33 field: forgerock.timestamp formats: - ISO8601 @@ -41,66 +49,81 @@ processors: timezone: UTC on_failure: - append: + tag: append_error_message_e1edf820 field: error.message value: '{{{ _ingest.on_failure_message }}}' # idm-access processing - set: + tag: set_event_type_24c7243c field: event.type value: [access] - convert: + tag: convert_forgerock_client_ip_to_client_ip_16f6010c field: forgerock.client.ip target_field: client.ip type: ip ignore_missing: true - set: + tag: set_client_port_43205056 field: client.port copy_from: forgerock.client.port ignore_failure: true - set: + tag: set_server_domain_654f70fc field: server.domain copy_from: forgerock.server.host ignore_failure: true - convert: + tag: convert_forgerock_server_ip_to_server_ip_b8435e6c field: forgerock.server.ip target_field: server.ip type: ip ignore_missing: true - set: + tag: set_user_id_374bda8c field: user.id copy_from: forgerock.userId ignore_failure: true - set: + tag: set_http_request_method_023d8f1e field: http.request.method copy_from: forgerock.http.request.method ignore_failure: true - set: + tag: set_http_request_Path_d840b8d2 field: http.request.Path copy_from: forgerock.http.request.path ignore_failure: true - convert: + tag: convert_forgerock_response_statusCode_to_http_response_status_code_f0bf401c target_field: http.response.status_code field: forgerock.response.statusCode type: integer ignore_failure: true - set: + tag: set_event_outcome_9d188049 if: ctx.forgerock?.response?.status == "SUCCESSFUL" field: event.outcome value: "success" ignore_failure: true - set: + tag: set_event_outcome_88b951f3 if: ctx.forgerock?.response?.status == "FAILED" field: event.outcome value: "failure" ignore_failure: true - convert: + tag: convert_forgerock_response_elapsedTime_1a480219 field: forgerock.response.elapsedTime type: long ignore_failure: true - set: + tag: set_event_duration_769e42e1 field: event.duration copy_from: forgerock.response.elapsedTime ignore_empty_value: true - script: + tag: script_37c8fd18 lang: painless if: ctx.event?.duration != null && ctx.forgerock?.response?.elapsedTimeUnits == 'MILLISECONDS' params: @@ -109,11 +132,13 @@ processors: ctx.event.duration *= params.MS_TO_NS; # cleaning up - remove: + tag: remove_ca120718 field: - json.timestamp - json.type ignore_failure: true - remove: + tag: remove_b800ac2e if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - json @@ -131,6 +156,7 @@ processors: ignore_failure: true ignore_missing: true - script: + tag: script_9d6fa8ba description: Drops null/empty values recursively lang: painless ignore_failure: true diff --git a/packages/forgerock/data_stream/idm_activity/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/idm_activity/elasticsearch/ingest_pipeline/default.yml index 244a98cde9c..2b0aa985f41 100644 --- a/packages/forgerock/data_stream/idm_activity/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/idm_activity/elasticsearch/ingest_pipeline/default.yml @@ -2,12 +2,15 @@ description: Pipeline for ForgeRock idm-activity audit logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - set: + tag: set_observer_vendor_a4252fc1 field: observer.vendor value: ForgeRock Identity Platform - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -20,20 +23,25 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - rename: + tag: rename_json_payload_to_forgerock_12e49639 field: json.payload target_field: forgerock - set: + tag: set_event_id_61b21b41 field: event.id copy_from: forgerock._id ignore_failure: true - set: + tag: set_transaction_id_04cf8970 field: transaction.id copy_from: forgerock.transactionId ignore_failure: true - date: + tag: date_forgerock_timestamp_to_timestamp_26758e33 field: forgerock.timestamp formats: - ISO8601 @@ -41,34 +49,41 @@ processors: timezone: UTC on_failure: - append: + tag: append_error_message_e1edf820 field: error.message value: '{{{ _ingest.on_failure_message }}}' # idm-activity processing - set: + tag: set_user_id_374bda8c field: user.id copy_from: forgerock.userId ignore_failure: true - set: + tag: set_user_effective_id_9b10f540 field: user.effective.id copy_from: forgerock.runAs ignore_failure: true - set: + tag: set_event_outcome_c2020dfc if: ctx.forgerock?.status == "SUCCESS" field: event.outcome value: "success" ignore_failure: true - set: + tag: set_event_outcome_14bef1b6 if: ctx.forgerock?.status == "FAIL" field: event.outcome value: "failure" ignore_failure: true # cleaning up - remove: + tag: remove_ca120718 field: - json.timestamp - json.type ignore_failure: true - remove: + tag: remove_6443d0af if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - json @@ -81,6 +96,7 @@ processors: ignore_failure: true ignore_missing: true - script: + tag: script_9d6fa8ba description: Drops null/empty values recursively lang: painless ignore_failure: true diff --git a/packages/forgerock/data_stream/idm_authentication/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/idm_authentication/elasticsearch/ingest_pipeline/default.yml index 6b9696af496..3d2522a939e 100644 --- a/packages/forgerock/data_stream/idm_authentication/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/idm_authentication/elasticsearch/ingest_pipeline/default.yml @@ -2,12 +2,15 @@ description: Pipeline for ForgeRock idm-authentication audit logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - set: + tag: set_observer_vendor_a4252fc1 field: observer.vendor value: ForgeRock Identity Platform - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -20,20 +23,25 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - rename: + tag: rename_json_payload_to_forgerock_12e49639 field: json.payload target_field: forgerock - set: + tag: set_event_id_61b21b41 field: event.id copy_from: forgerock._id ignore_failure: true - set: + tag: set_transaction_id_04cf8970 field: transaction.id copy_from: forgerock.transactionId ignore_failure: true - date: + tag: date_forgerock_timestamp_to_timestamp_26758e33 field: forgerock.timestamp formats: - ISO8601 @@ -41,33 +49,40 @@ processors: timezone: UTC on_failure: - append: + tag: append_error_message_e1edf820 field: error.message value: '{{{ _ingest.on_failure_message }}}' # idm-authentication processing - set: + tag: set_event_category_e43d8992 field: event.category value: [authentication] - set: + tag: set_user_id_374bda8c field: user.id copy_from: forgerock.userId ignore_failure: true - set: + tag: set_event_outcome_9c5049a6 if: ctx.forgerock?.result == "SUCCESSFUL" field: event.outcome value: "success" ignore_failure: true - set: + tag: set_event_outcome_e25c8538 if: ctx.forgerock?.result == "FAILED" field: event.outcome value: "failure" ignore_failure: true # cleaning up - remove: + tag: remove_ca120718 field: - json.timestamp - json.type ignore_failure: true - remove: + tag: remove_e92017c7 if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - json @@ -91,6 +106,7 @@ processors: ignore_failure: true ignore_missing: true - script: + tag: script_9d6fa8ba description: Drops null/empty values recursively lang: painless ignore_failure: true diff --git a/packages/forgerock/data_stream/idm_config/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/idm_config/elasticsearch/ingest_pipeline/default.yml index 56af2e8c972..8ac0713bca0 100644 --- a/packages/forgerock/data_stream/idm_config/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/idm_config/elasticsearch/ingest_pipeline/default.yml @@ -2,12 +2,15 @@ description: Pipeline for ForgeRock idm-config audit logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - set: + tag: set_observer_vendor_a4252fc1 field: observer.vendor value: ForgeRock Identity Platform - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -20,20 +23,25 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - rename: + tag: rename_json_payload_to_forgerock_12e49639 field: json.payload target_field: forgerock - set: + tag: set_event_id_61b21b41 field: event.id copy_from: forgerock._id ignore_failure: true - set: + tag: set_transaction_id_04cf8970 field: transaction.id copy_from: forgerock.transactionId ignore_failure: true - date: + tag: date_forgerock_timestamp_to_timestamp_26758e33 field: forgerock.timestamp formats: - ISO8601 @@ -41,27 +49,33 @@ processors: timezone: UTC on_failure: - append: + tag: append_error_message_e1edf820 field: error.message value: '{{{ _ingest.on_failure_message }}}' # idm-config processing - set: + tag: set_event_category_e56470ca field: event.category value: [configuration] - set: + tag: set_user_id_374bda8c field: user.id copy_from: forgerock.userId ignore_failure: true - set: + tag: set_user_effective_id_9b10f540 field: user.effective.id copy_from: forgerock.runAs ignore_failure: true # cleaning up - remove: + tag: remove_ca120718 field: - json.timestamp - json.type ignore_failure: true - remove: + tag: remove_e92017c7 if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - json @@ -85,6 +99,7 @@ processors: ignore_failure: true ignore_missing: true - script: + tag: script_9d6fa8ba description: Drops null/empty values recursively lang: painless ignore_failure: true diff --git a/packages/forgerock/data_stream/idm_core/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/idm_core/elasticsearch/ingest_pipeline/default.yml index 0807d519c33..15f5b0cec54 100644 --- a/packages/forgerock/data_stream/idm_core/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/idm_core/elasticsearch/ingest_pipeline/default.yml @@ -2,12 +2,15 @@ description: Pipeline for ForgeRock idm-core debug logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - set: + tag: set_observer_vendor_a4252fc1 field: observer.vendor value: ForgeRock Identity Platform - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -20,9 +23,11 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_forgerock_09502332 field: event.original target_field: forgerock - date: + tag: date_forgerock_timestamp_to_timestamp_26758e33 field: forgerock.timestamp formats: - ISO8601 @@ -30,39 +35,47 @@ processors: timezone: UTC on_failure: - append: + tag: append_error_message_e1edf820 field: error.message value: '{{{ _ingest.on_failure_message }}}' - set: + tag: set_event_reason_e343387e field: event.reason copy_from: forgerock.payload ignore_failure: true if: ctx.forgerock?.payload instanceof String - set: + tag: set_event_reason_e775b008 field: event.reason copy_from: forgerock.payload.message ignore_empty_value: true # cleaning up - remove: + tag: remove_1d2e828d field: - forgerock.timestamp - forgerock.type - forgerock.source ignore_failure: true - remove: + tag: remove_b19558b7 if: ctx.forgerock?.payload instanceof String && (ctx.tags == null || !ctx.tags.contains('preserve_duplicate_custom_fields')) field: - forgerock.payload ignore_missing: true - remove: + tag: remove_fe928479 if: ctx.forgerock?.payload instanceof Map && (ctx.tags == null || !ctx.tags.contains('preserve_duplicate_custom_fields')) field: - forgerock.payload.message ignore_missing: true - rename: + tag: rename_forgerock_payload_to_forgerock_idm_core_f018ef72 field: forgerock.payload target_field: forgerock.idm_core ignore_missing: true - script: + tag: script_9d6fa8ba description: Drops null/empty values recursively lang: painless ignore_failure: true diff --git a/packages/forgerock/data_stream/idm_sync/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/idm_sync/elasticsearch/ingest_pipeline/default.yml index 2ef59cd93b0..b4c3b403b56 100644 --- a/packages/forgerock/data_stream/idm_sync/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/idm_sync/elasticsearch/ingest_pipeline/default.yml @@ -2,12 +2,15 @@ description: Pipeline for ForgeRock idm-sync audit logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - set: + tag: set_observer_vendor_a4252fc1 field: observer.vendor value: ForgeRock Identity Platform - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -20,20 +23,25 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - rename: + tag: rename_json_payload_to_forgerock_12e49639 field: json.payload target_field: forgerock - set: + tag: set_event_id_61b21b41 field: event.id copy_from: forgerock._id ignore_failure: true - set: + tag: set_transaction_id_04cf8970 field: transaction.id copy_from: forgerock.transactionId ignore_failure: true - date: + tag: date_forgerock_timestamp_to_timestamp_26758e33 field: forgerock.timestamp formats: - ISO8601 @@ -41,50 +49,60 @@ processors: timezone: UTC on_failure: - append: + tag: append_error_message_e1edf820 field: error.message value: '{{{ _ingest.on_failure_message }}}' # idm-sync processing - set: + tag: set_user_id_374bda8c field: user.id copy_from: forgerock.userId ignore_failure: true - set: + tag: set_event_outcome_c2020dfc if: ctx.forgerock?.status == "SUCCESS" field: event.outcome value: "success" ignore_failure: true - set: + tag: set_event_outcome_14bef1b6 if: ctx.forgerock?.status == "FAIL" field: event.outcome value: "failure" ignore_failure: true - remove: + tag: remove_13c442d6 if: ctx.forgerock?.exception == null field: - forgerock.exception ignore_failure: true - remove: + tag: remove_0aa28762 if: ctx.forgerock?.message == null field: - forgerock.message ignore_failure: true - remove: + tag: remove_860192d8 if: ctx.forgerock?.sourceObjectId == null field: - forgerock.sourceObjectId ignore_failure: true - remove: + tag: remove_68afc71c if: ctx.forgerock?.targetObjectId == null field: - forgerock.targetObjectId ignore_failure: true # cleaning up - remove: + tag: remove_ca120718 field: - json.timestamp - json.type ignore_failure: true - remove: + tag: remove_e92017c7 if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - json @@ -108,6 +126,7 @@ processors: ignore_failure: true ignore_missing: true - script: + tag: script_9d6fa8ba description: Drops null/empty values recursively lang: painless ignore_failure: true diff --git a/packages/forgerock/manifest.yml b/packages/forgerock/manifest.yml index 9abd0b7fc1a..9106efe7b51 100644 --- a/packages/forgerock/manifest.yml +++ b/packages/forgerock/manifest.yml @@ -1,6 +1,6 @@ name: forgerock title: "ForgeRock" -version: "1.24.1" +version: "1.24.2" description: Collect audit logs from ForgeRock with Elastic Agent. type: integration format_version: "3.3.2" diff --git a/packages/gcp/changelog.yml b/packages/gcp/changelog.yml index d1dbab5ab4e..bd6130a7ea7 100644 --- a/packages/gcp/changelog.yml +++ b/packages/gcp/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.50.3" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "2.50.2" changes: - description: Add security category to the Load Balancing and VPC Flow data streams to match their policy template categories. diff --git a/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index ae20e05261b..211ca268f63 100644 --- a/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -3,54 +3,64 @@ description: Pipeline for Google Cloud audit logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: 'ctx.event?.original == null' - json: + tag: json_event_original_to_json_da2067de field: event.original target_field: json if: ctx.event?.original != null on_failure: # Parse cluster notification on failure - set: + tag: set_gcp_audit_notification_232db967 field: gcp.audit.notification copy_from: event.original ignore_empty_value: true - set: + tag: set_gcp_audit_type_c033c617 field: gcp.audit.type copy_from: "json.protoPayload.@type" ignore_failure: true -## -# https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry -# protoPayload @type must be type.googleapis.com/google.cloud.audit.AuditLog -## + ## + # https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry + # protoPayload @type must be type.googleapis.com/google.cloud.audit.AuditLog + ## - drop: + tag: drop_10e9b810 description: Drop the document if it is not of AuditLog type if: ctx.gcp?.audit?.type != null && ctx.gcp?.audit?.type != 'type.googleapis.com/google.cloud.audit.AuditLog' -# .insertId + # .insertId - set: + tag: set_event_id_008e62b2 field: event.id copy_from: json.insertId if: ctx.json?.insertId != null -# .logName + # .logName - rename: + tag: rename_json_logName_to_log_logger_73f0a6eb field: json.logName target_field: log.logger ignore_missing: true -# .severity + # .severity - rename: + tag: rename_json_severity_to_log_level_db5923df field: json.severity target_field: log.level ignore_missing: true -## -# Extract the type of audit logging data from logName to event.provider -# https://cloud.google.com/pubsub/docs/audit-logging#log_name -## + ## + # Extract the type of audit logging data from logName to event.provider + # https://cloud.google.com/pubsub/docs/audit-logging#log_name + ## - dissect: + tag: dissect_log_logger_c8ea0152 field: log.logger pattern: "%{}%2F%{event.provider}" ignore_missing: true @@ -58,55 +68,65 @@ processors: ignore_failure: true - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_cloud_provider_62c5e42e field: cloud.provider value: gcp - date: + tag: date_json_timestamp_7a63aef5 field: json.timestamp timezone: UTC formats: - ISO8601 if: ctx.json?.timestamp != null && ctx.json.timestamp != '' -## -# MonitoredResource -# .resource -# MonitoredResource https://cloud.google.com/logging/docs/reference/v2/rest/v2/MonitoredResource -## + ## + # MonitoredResource + # .resource + # MonitoredResource https://cloud.google.com/logging/docs/reference/v2/rest/v2/MonitoredResource + ## - set: + tag: set_cloud_project_id_121ba626 field: cloud.project.id copy_from: json.resource.labels.project_id if: ctx.json?.resource?.labels?.project_id != null - set: + tag: set_cloud_instance_id_e0be3fd4 field: cloud.instance.id copy_from: json.resource.labels.instance_id if: ctx.json?.resource?.labels?.instance_id != null - set: + tag: set_cloud_availability_zone_24b38f5a field: cloud.availability_zone copy_from: json.resource.labels.location ignore_empty_value: true -## -# MonitoredResourceDescriptor type -# https://cloud.google.com/logging/docs/reference/v2/rest/v2/monitoredResourceDescriptors/list#MonitoredResourceDescriptor -# resource list values https://cloud.google.com/logging/docs/api/v2/resource-list -## + ## + # MonitoredResourceDescriptor type + # https://cloud.google.com/logging/docs/reference/v2/rest/v2/monitoredResourceDescriptors/list#MonitoredResourceDescriptor + # resource list values https://cloud.google.com/logging/docs/api/v2/resource-list + ## - set: + tag: set_orchestrator_type_757229a9 field: orchestrator.type value: kubernetes if: ctx.json?.resource?.type != null && (ctx.json?.resource?.type == 'k8s_cluster' || ctx.json?.resource?.type == 'gke_cluster') - set: + tag: set_orchestrator_cluster_name_83531fb4 field: orchestrator.cluster.name copy_from: json.resource.labels.cluster_name ignore_empty_value: true if: ctx.json?.resource?.type != null && (ctx.json?.resource?.type == 'k8s_cluster' || ctx.json?.resource?.type == 'gke_cluster') - set: + tag: set__temp_type_3c87c5ae field: _temp.type copy_from: json.protoPayload.resourceName ignore_empty_value: true if: ctx.json?.resource?.type != null && ctx.json?.resource?.type == 'k8s_cluster' - grok: + tag: grok__temp_type_4273d731 field: _temp.type patterns: - '%{DATA}/%{API_VERSION:orchestrator.api_version}/namespaces/%{DATA:orchestrator.namespace}/%{RESOURCE_TYPE:orchestrator.resource.type}(/%{HOSTNAME:orchestrator.resource.name})?' @@ -119,10 +139,12 @@ processors: RESOURCE_TYPE: ([a-z]+((\.[a-z0-9]+)+)?) ignore_missing: true - script: + tag: script_13bc28fc description: Appends any relevant entity to `related.entity` for all events lang: painless on_failure: - set: + tag: set_error_message_679c76c6 description: Adds error reason to the document field: error.message value: "{{{ _ingest.on_failure_message }}}" @@ -209,33 +231,39 @@ processors: ctx.related.entity = entities; } -## -# Extract origin, target, and related entity IDs -## + ## + # Extract origin, target, and related entity IDs + ## - append: + tag: append_actor_entity_id_8c5df9c1 field: actor.entity.id value: "{{{json.protoPayload.authenticationInfo.principalSubject}}}" if: ctx.json?.protoPayload?.authenticationInfo?.principalSubject != null - append: + tag: append_actor_entity_id_5873a061 field: actor.entity.id value: "{{{json.protoPayload.authenticationInfo.principalEmail}}}" if: ctx.json?.protoPayload?.authenticationInfo?.principalEmail != null - append: + tag: append_target_entity_id_a8c26482 field: target.entity.id value: "{{{json.protoPayload.resourceName}}}" if: ctx.json?.resource?.type != 'k8s_cluster' && ctx.json?.protoPayload?.resourceName != null - append: + tag: append_target_entity_id_e0955b84 field: target.entity.id value: "{{{json.resource.disk_id}}}" if: ctx.json?.resouce?.disk_id != null -## -# Classify actor and target entities by type for enhanced graph visualization -## + ## + # Classify actor and target entities by type for enhanced graph visualization + ## - script: + tag: script_0b5ddf1c description: Classify actor and target entities into type-specific fields lang: painless on_failure: - set: + tag: set_error_message_f032c8ea description: Adds error reason to the document field: error.message value: "{{{ _ingest.on_failure_message }}}" @@ -349,10 +377,12 @@ processors: } } - script: + tag: script_13bc28fc description: Appends any relevant entity to `related.entity` for all events lang: painless on_failure: - set: + tag: set_error_message_679c76c6 description: Adds error reason to the document field: error.message value: "{{{ _ingest.on_failure_message }}}" @@ -439,32 +469,38 @@ processors: ctx.related.entity = entities; } -## -# AuthenticationInfo -# .protoPayload.authenticationInfo -# https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog#AuthenticationInfo -## + ## + # AuthenticationInfo + # .protoPayload.authenticationInfo + # https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog#AuthenticationInfo + ## - rename: + tag: rename_json_protoPayload_authenticationInfo_authoritySelector_to_gcp_audit_authentication_info_authority_selector_8503f1a1 field: json.protoPayload.authenticationInfo.authoritySelector target_field: gcp.audit.authentication_info.authority_selector ignore_failure: true - rename: + tag: rename_json_protoPayload_authenticationInfo_principalEmail_to_gcp_audit_authentication_info_principal_email_9739641b field: json.protoPayload.authenticationInfo.principalEmail target_field: gcp.audit.authentication_info.principal_email ignore_failure: true - rename: + tag: rename_json_protoPayload_authenticationInfo_principalSubject_to_gcp_audit_authentication_info_principal_subject_0d76993b field: json.protoPayload.authenticationInfo.principalSubject target_field: gcp.audit.authentication_info.principal_subject ignore_failure: true - rename: + tag: rename_json_protoPayload_authenticationInfo_serviceAccountKeyName_to_gcp_audit_authentication_info_service_account_key_name_dfd8d4d3 field: json.protoPayload.authenticationInfo.serviceAccountKeyName target_field: gcp.audit.authentication_info.service_account_key_name ignore_failure: true - rename: + tag: rename_json_protoPayload_authenticationInfo_serviceAccountDelegationInfo_to_gcp_audit_authentication_info_service_account_delegation_info_883b4747 field: json.protoPayload.authenticationInfo.serviceAccountDelegationInfo target_field: gcp.audit.authentication_info.service_account_delegation_info ignore_failure: true - rename: + tag: rename_json_protoPayload_authenticationInfo_thirdPartyPrincipal_to_gcp_audit_authentication_info_third_party_principal_19aae662 field: json.protoPayload.authenticationInfo.thirdPartyPrincipal target_field: gcp.audit.authentication_info.third_party_principal ignore_failure: true @@ -472,6 +508,7 @@ processors: # email address of authenticated user (redacted) or service account # principal_email -> client.user.email - rename: + tag: rename_gcp_audit_authentication_info_principal_email_to_client_user_email_d32fc7d2 field: gcp.audit.authentication_info.principal_email target_field: client.user.email if: ctx.client?.user?.email == null @@ -479,21 +516,24 @@ processors: # identity of requesting first or third party # principal_subject -> client.user.id - rename: + tag: rename_gcp_audit_authentication_info_principal_subject_to_client_user_id_603af332 field: gcp.audit.authentication_info.principal_subject target_field: client.user.id if: ctx.client?.user?.id == null ignore_missing: true -## -# AuthorizationInfo -# .protoPayload.authorizationInfo -# https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog#authorizationinfo -## + ## + # AuthorizationInfo + # .protoPayload.authorizationInfo + # https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog#authorizationinfo + ## - set: + tag: set_gcp_audit_authorization_info_54848940 field: gcp.audit.authorization_info copy_from: json.protoPayload.authorizationInfo ignore_failure: true - foreach: + tag: foreach_gcp_audit_authorization_info_4f513c47 field: gcp.audit.authorization_info ignore_missing: true ignore_failure: true @@ -503,11 +543,12 @@ processors: target_field: _ingest._value.resource_attributes if: ctx?.gcp?.audit?.authorization_info != null && ctx?.gcp?.audit?.authorization_info instanceof List -## -# Labels -# .labels -## + ## + # Labels + # .labels + ## - set: + tag: set_gcp_audit_labels_a7884819 field: gcp.audit.labels copy_from: json.labels if: ctx.json?.labels != null @@ -517,6 +558,7 @@ processors: if: ctx.labels?.payload != null on_failure: - append: + tag: append_error_message_c858cf42 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -554,61 +596,69 @@ processors: override: false if: ctx.orchestrator?.resource?.name == null && ctx.labels?.payload?.resource != null -## -# RequestMetadata -# .protoPayload.requestMetadata -# https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog#requestmetadata -## + ## + # RequestMetadata + # .protoPayload.requestMetadata + # https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog#requestmetadata + ## - convert: + tag: convert_json_protoPayload_requestMetadata_callerIp_to_source_ip_f3f594de if: ctx.json?.protoPayload?.requestMetadata?.callerIp != null && ctx.json?.protoPayload?.requestMetadata?.callerIp != "gce-internal-ip" && ctx.json?.protoPayload?.requestMetadata?.callerIp != "private" type: ip field: json.protoPayload.requestMetadata.callerIp target_field: source.ip ignore_missing: true - set: + tag: set_user_agent_original_553976c5 field: user_agent.original copy_from: json.protoPayload.requestMetadata.callerSuppliedUserAgent ignore_failure: true -## -# Metadata -# .protoPayload.metadata -# https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog -## + ## + # Metadata + # .protoPayload.metadata + # https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog + ## - set: + tag: set_gcp_audit_metadata_bd142044 field: gcp.audit.metadata copy_from: json.protoPayload.metadata if: ctx.json?.protoPayload?.metadata != null -## -# PolicyViolationInfo -# .protoPayload.orgPolicyViolationInfo -# https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog#policyviolationinfo -## + ## + # PolicyViolationInfo + # .protoPayload.orgPolicyViolationInfo + # https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog#policyviolationinfo + ## - set: + tag: set_gcp_audit_policy_violation_info_violations_f6f04e9b field: gcp.audit.policy_violation_info.violations copy_from: json.protoPayload.policyViolationInfo.orgPolicyViolationInfo.violationInfo ignore_failure: true - rename: + tag: rename_json_protoPayload_policyViolationInfo_orgPolicyViolationInfo_payload_to_gcp_audit_policy_violation_info_payload_5791a15c field: json.protoPayload.policyViolationInfo.orgPolicyViolationInfo.payload target_field: gcp.audit.policy_violation_info.payload ignore_failure: true - rename: + tag: rename_json_protoPayload_policyViolationInfo_orgPolicyViolationInfo_resourceType_to_gcp_audit_policy_violation_info_resource_type_bec412c9 field: json.protoPayload.policyViolationInfo.orgPolicyViolationInfo.resourceType target_field: gcp.audit.policy_violation_info.resource_type ignore_failure: true - rename: + tag: rename_json_protoPayload_policyViolationInfo_orgPolicyViolationInfo_resourceTags_to_gcp_audit_policy_violation_info_resource_tags_5d197073 field: json.protoPayload.policyViolationInfo.orgPolicyViolationInfo.resourceTags target_field: gcp.audit.policy_violation_info.resource_tags ignore_failure: true -## -# LogEntryOperation -# .operation -# https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#logentryoperation -## -# set only if it is not the same as insertId + ## + # LogEntryOperation + # .operation + # https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#logentryoperation + ## + # set only if it is not the same as insertId - set: + tag: set_gcp_audit_logentry_operation_id_162dd8d0 field: gcp.audit.logentry_operation.id copy_from: json.operation.id if: ctx.json?.operation?.id != null && ctx.event?.id != null && ctx.event?.id != ctx.json?.operation?.id @@ -637,161 +687,195 @@ processors: ctx.event.type.add('end'); } -## -# BindingDelta -# protPayload.serviceData.policyDelta.bindingDeltas -# https://docs.cloud.google.com/iam/docs/reference/rpc/google.iam.v1#google.iam.v1.BindingDelta -## + ## + # BindingDelta + # protPayload.serviceData.policyDelta.bindingDeltas + # https://docs.cloud.google.com/iam/docs/reference/rpc/google.iam.v1#google.iam.v1.BindingDelta + ## - set: + tag: set_gcp_audit_service_data_policy_delta_binding_deltas_9a07e6ba field: gcp.audit.service_data.policy_delta.binding_deltas copy_from: json.protoPayload.serviceData.policyDelta.bindingDeltas if: ctx.json?.protoPayload?.serviceData?.policyDelta?.bindingDeltas != null && !ctx.json.protoPayload.serviceData.policyDelta.bindingDeltas.isEmpty() -# TODO remove duplicate protoPayload.methodName + # TODO remove duplicate protoPayload.methodName - set: + tag: set_event_action_88429a42 field: event.action copy_from: json.protoPayload.methodName ignore_failure: true - convert: + tag: convert_json_protoPayload_numResponseItems_to_gcp_audit_num_response_items_7293b658 field: json.protoPayload.numResponseItems target_field: gcp.audit.num_response_items type: long ignore_missing: true - set: + tag: set_gcp_audit_request_8473ca70 field: gcp.audit.request copy_from: json.protoPayload.request if: ctx.json?.protoPayload?.request != null - remove: + tag: remove_gcp_audit_request_policy_ec3e5101 field: gcp.audit.request.policy ignore_missing: true if: ctx.json?.protoPayload?.request?.policy != null && !(ctx.json?.protoPayload?.request.policy instanceof Map) - set: + tag: set_gcp_audit_request_policy_value_27b9f5c0 field: gcp.audit.request.policy_value copy_from: json.protoPayload.request.policy ignore_failure: true if: ctx.json?.protoPayload?.request?.policy != null && !(ctx.json?.protoPayload?.request.policy instanceof Map) - set: + tag: set_gcp_audit_response_9970acba field: gcp.audit.response copy_from: json.protoPayload.response if: ctx.json?.protoPayload?.response != null - remove: + tag: remove_gcp_audit_response_status_b64de13d field: gcp.audit.response.status ignore_missing: true if: ctx.json?.protoPayload?.response?.status != null && !(ctx.json?.protoPayload?.response.status instanceof Map) - set: + tag: set_gcp_audit_response_status_value_141d9fd6 field: gcp.audit.response.status_value copy_from: json.protoPayload.response.status ignore_failure: true if: ctx.json?.protoPayload?.response?.status != null && !(ctx.json?.protoPayload?.response.status instanceof Map) - set: + tag: set_gcp_audit_resource_name_9ff21a1e field: gcp.audit.resource_name copy_from: json.protoPayload.resourceName ignore_failure: true - set: + tag: set_gcp_audit_resource_location_current_locations_26959615 field: gcp.audit.resource_location.current_locations copy_from: json.protoPayload.resourceLocation.currentLocations ignore_failure: true - set: + tag: set_gcp_audit_resource_labels_resource_container_6146614e field: gcp.audit.resource.labels.resource_container copy_from: json.resource.labels.resource_container ignore_empty_value: true - set: + tag: set_gcp_audit_resource_type_db73d47a field: gcp.audit.resource.type copy_from: json.resource.type ignore_empty_value: true - set: + tag: set_gcp_audit_service_name_791764ca field: gcp.audit.service_name copy_from: json.protoPayload.serviceName ignore_failure: true - rename: + tag: rename_gcp_audit_service_name_to_service_name_2a1af9c2 field: gcp.audit.service_name target_field: service.name if: ctx.service?.name == null ignore_missing: true -# Handle JsonPayload Fields + # Handle JsonPayload Fields - date: + tag: date_json_receiveTimestamp_to_gcp_audit_receive_timestamp_8772c0b0 field: json.receiveTimestamp target_field: gcp.audit.receive_timestamp formats: - ISO8601 if: ctx.json?.receiveTimestamp != null && ctx.json.receiveTimestamp != '' - convert: + tag: convert_json_jsonPayload_access_callerIp_to_source_ip_3af84c77 if: ctx.json?.jsonPayload?.access?.callerIp != '' field: json.jsonPayload.access.callerIp target_field: source.ip type: ip ignore_missing: true - rename: + tag: rename_json_jsonPayload_access_callerIpGeo_regionCode_to_gcp_audit_access_caller_ip_geo_region_code_416da0b6 field: json.jsonPayload.access.callerIpGeo.regionCode target_field: gcp.audit.access.caller_ip_geo.region_code ignore_missing: true - rename: + tag: rename_json_jsonPayload_access_methodName_to_gcp_audit_access_method_name_67af4c66 field: json.jsonPayload.access.methodName target_field: gcp.audit.access.method_name ignore_missing: true - set: + tag: set_event_action_8d52ad52 field: event.action copy_from: gcp.audit.access.method_name override: false ignore_empty_value: true - rename: + tag: rename_json_jsonPayload_access_principalEmail_to_gcp_audit_access_principal_email_16efa92a field: json.jsonPayload.access.principalEmail target_field: gcp.audit.access.principal_email ignore_missing: true - set: + tag: set_client_user_email_4f97db12 field: client.user.email copy_from: gcp.audit.access.principal_email override: false ignore_empty_value: true - rename: + tag: rename_json_jsonPayload_access_principalSubject_to_gcp_audit_access_principal_subject_622208c2 field: json.jsonPayload.access.principalSubject target_field: gcp.audit.access.principal_subject ignore_missing: true - rename: + tag: rename_json_jsonPayload_access_serviceName_to_gcp_audit_access_service_name_68e858ce field: json.jsonPayload.access.serviceName target_field: gcp.audit.access.service_name ignore_missing: true - set: + tag: set_service_name_6cd2eeb4 field: service.name copy_from: gcp.audit.access.service_name override: false ignore_empty_value: true - rename: + tag: rename_json_jsonPayload_access_userAgent_to_gcp_audit_access_user_agent_a407d9d4 field: json.jsonPayload.access.userAgent target_field: gcp.audit.access.user_agent ignore_missing: true - set: + tag: set_user_agent_original_7c37584a field: user_agent.original copy_from: gcp.audit.access.user_agent override: false ignore_empty_value: true - user_agent: + tag: user_agent_user_agent_original_b5325863 field: user_agent.original ignore_missing: true - date: + tag: date_json_jsonPayload_actionTime_to_gcp_audit_action_time_be349749 field: json.jsonPayload.actionTime target_field: gcp.audit.action_time formats: - ISO8601 if: ctx.json?.jsonPayload?.actionTime != null && ctx.json.jsonPayload.actionTime != '' - rename: + tag: rename_json_jsonPayload_actionType_to_gcp_audit_action_type_75969e80 field: json.jsonPayload.actionType target_field: gcp.audit.action_type ignore_missing: true - rename: + tag: rename_json_jsonPayload_affectedResources_to_gcp_audit_affected_resources_76dba6aa field: json.jsonPayload.affectedResources target_field: gcp.audit.affected_resources ignore_missing: true - rename: + tag: rename_json_jsonPayload_learnMoreUri_to_gcp_audit_learn_more_uri_a9f88959 field: json.jsonPayload.learnMoreUri target_field: gcp.audit.learn_more_uri ignore_missing: true - rename: + tag: rename_json_jsonPayload_sourceLogIds_to_gcp_audit_source_log_ids_05815a31 field: json.jsonPayload.sourceLogIds target_field: gcp.audit.source_log_ids ignore_missing: true - foreach: + tag: foreach_gcp_audit_source_log_ids_7b722f0a field: gcp.audit.source_log_ids if: ctx.gcp?.audit?.source_log_ids instanceof List ignore_missing: true @@ -801,6 +885,7 @@ processors: target_field: _ingest._value.insert_id ignore_missing: true - foreach: + tag: foreach_gcp_audit_source_log_ids_3120fb56 field: gcp.audit.source_log_ids if: ctx.gcp?.audit?.source_log_ids instanceof List ignore_missing: true @@ -810,6 +895,7 @@ processors: target_field: _ingest._value.query_uri ignore_missing: true - foreach: + tag: foreach_gcp_audit_source_log_ids_a6c73328 field: gcp.audit.source_log_ids if: ctx.gcp?.audit?.source_log_ids instanceof List ignore_missing: true @@ -819,6 +905,7 @@ processors: target_field: _ingest._value.resource_container ignore_missing: true - foreach: + tag: foreach_gcp_audit_source_log_ids_fec43cee field: gcp.audit.source_log_ids if: ctx.gcp?.audit?.source_log_ids instanceof List ignore_missing: true @@ -826,13 +913,14 @@ processors: date: field: _ingest._value.logTime target_field: _ingest._value.log_time - formats: + formats: - ISO8601 on_failure: - remove: field: _ingest._value.logTime ignore_missing: true - foreach: + tag: foreach_gcp_audit_source_log_ids_c49308d5 field: gcp.audit.source_log_ids if: ctx.gcp?.audit?.source_log_ids instanceof List ignore_missing: true @@ -840,74 +928,87 @@ processors: remove: field: _ingest._value.logTime ignore_missing: true -## -# .protoPayload.Status -# https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog#Status -# google.rpc.Code referred in Status can have the following values -# https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto -## + ## + # .protoPayload.Status + # https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog#Status + # google.rpc.Code referred in Status can have the following values + # https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto + ## - convert: + tag: convert_json_protoPayload_status_code_to_gcp_audit_status_code_0ad56d96 field: json.protoPayload.status.code target_field: gcp.audit.status.code type: long ignore_missing: true - set: + tag: set_gcp_audit_status_message_904fe43f field: gcp.audit.status.message copy_from: json.protoPayload.status.message ignore_failure: true - set: + tag: set_gcp_audit_status_details_9e97d7cc field: gcp.audit.status.details copy_from: json.protoPayload.status.details ignore_empty_value: true - set: + tag: set_event_outcome_ac53d779 field: event.outcome value: success if: ctx.gcp?.audit?.status?.code != null && ctx.gcp?.audit?.status?.code == 0 - set: + tag: set_event_outcome_d883f0c2 field: event.outcome value: failure if: ctx?.gcp?.audit?.status?.code != null && ctx?.gcp?.audit?.status?.code != 0 - set: + tag: set_event_outcome_2ceef6ae field: event.outcome value: success if: ctx?.gcp?.audit?.status?.code == null && ctx?.gcp?.audit?.authorization_info != null && ctx?.gcp?.audit?.authorization_info instanceof List && ctx?.gcp?.audit?.authorization_info.size() == 1 && ctx?.gcp?.audit?.authorization_info[0]?.granted == true - set: + tag: set_event_outcome_d1bb2700 field: event.outcome value: failure if: ctx?.gcp?.audit?.status?.code == null && ctx?.gcp?.audit?.authorization_info != null && ctx?.gcp?.audit?.authorization_info instanceof List && ctx?.gcp?.audit?.authorization_info.size() == 1 && ctx?.gcp?.audit?.authorization_info[0]?.granted == false - set: + tag: set_event_outcome_2820f2e9 field: event.outcome value: unknown override: false -## -# if gcp.audit.authorization_info.[0].granted is true then -# set event.category [network, configuration] and event.type to [access, allowed]; -# Caveat -# 1. protoPayload.resourceName is a single value while authorization_info[].resource -# is a list. -# 2. as per test data authorization_info may not be as per spec. -## + ## + # if gcp.audit.authorization_info.[0].granted is true then + # set event.category [network, configuration] and event.type to [access, allowed]; + # Caveat + # 1. protoPayload.resourceName is a single value while authorization_info[].resource + # is a list. + # 2. as per test data authorization_info may not be as per spec. + ## - append: + tag: append_event_category_0236568c field: event.category value: ['network', 'configuration'] if: ctx?.gcp?.audit?.authorization_info != null && ctx?.gcp?.audit?.authorization_info instanceof List && ctx?.gcp?.audit?.authorization_info.size() == 1 - append: + tag: append_event_type_f6ee9b9b field: event.type value: ['access', 'allowed'] if: ctx?.gcp?.audit?.authorization_info != null && ctx?.gcp?.audit?.authorization_info instanceof List && ctx?.gcp?.audit?.authorization_info.size() == 1 && ctx?.gcp?.audit?.authorization_info[0]?.granted != null && ctx?.gcp?.audit?.authorization_info[0]?.granted - append: + tag: append_event_type_c3c30df1 field: event.type value: ['access', 'denied'] if: ctx?.gcp?.audit?.authorization_info != null && ctx?.gcp?.audit?.authorization_info instanceof List && ctx?.gcp?.audit?.authorization_info.size() == 1 && ctx?.gcp?.audit?.authorization_info[0]?.granted != null && !ctx?.gcp?.audit?.authorization_info[0]?.granted # IP Geolocation Lookup - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true # IP Autonomous System (AS) Lookup - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -916,49 +1017,58 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true -# Populate user.email from client.user.email + # Populate user.email from client.user.email - set: + tag: set_user_email_53a16970 field: user.email copy_from: client.user.email ignore_empty_value: true -# Populate user.id from client.user.id + # Populate user.id from client.user.id - set: + tag: set_user_id_89112192 field: user.id copy_from: client.user.id ignore_empty_value: true -# Related fields + # Related fields - append: + tag: append_related_ip_feeba47d field: related.ip value: "{{{source.ip}}}" allow_duplicates: false - append: + tag: append_related_user_fd6114de field: related.user value: "{{{client.user.email}}}" if: ctx.client?.user?.email != null allow_duplicates: false -## -# clean-up -## + ## + # clean-up + ## - rename: + tag: rename_json_to_gcp_audit_flattened_2f590e53 field: json target_field: gcp.audit.flattened if: ctx.json != null && ctx._conf?.keep_json == true - remove: + tag: remove_d60b84b0 field: - _conf - _temp - json ignore_missing: true - script: + tag: script_0d92a93d description: Drops null and empty values and dotted keys recursively lang: painless source: | @@ -993,3 +1103,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/gcp/data_stream/billing/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/billing/elasticsearch/ingest_pipeline/default.yml index 90f3e8699a9..f4a03305f06 100644 --- a/packages/gcp/data_stream/billing/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/billing/elasticsearch/ingest_pipeline/default.yml @@ -2,10 +2,12 @@ description: Pipeline for parsing GCP Billing metrics. processors: - set: + tag: set_event_start_ada05dda field: event.start copy_from: "gcp.billing.usage_start_time" ignore_empty_value: true - set: + tag: set_event_end_30d22bb4 field: event.end copy_from: "gcp.billing.usage_end_time" ignore_empty_value: true diff --git a/packages/gcp/data_stream/cloudrun_metrics/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/cloudrun_metrics/elasticsearch/ingest_pipeline/default.yml index fac7ac1f9b1..97183734f32 100644 --- a/packages/gcp/data_stream/cloudrun_metrics/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/cloudrun_metrics/elasticsearch/ingest_pipeline/default.yml @@ -2,55 +2,68 @@ description: Pipeline for parsing GCP Cloud Run metrics. processors: - fingerprint: + tag: fingerprint_5c1d1252 fields: - gcp.labels target_field: gcp.labels_fingerprint ignore_missing: true - rename: + tag: rename_gcp_metrics_container_billable_instance_time_value_to_gcp_cloudrun_metrics_container_billable_instance_time_208cc520 field: gcp.metrics.container.billable_instance_time.value target_field: gcp.cloudrun_metrics.container.billable_instance_time ignore_missing: true - rename: + tag: rename_gcp_metrics_container_cpu_allocation_time_value_to_gcp_cloudrun_metrics_container_cpu_allocation_time_sec_b23e2f41 field: gcp.metrics.container.cpu.allocation_time.value target_field: gcp.cloudrun_metrics.container.cpu.allocation_time.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_container_cpu_utilizations_value_to_gcp_cloudrun_metrics_container_cpu_utilizations_9ff4fab0 field: gcp.metrics.container.cpu.utilizations.value target_field: gcp.cloudrun_metrics.container.cpu.utilizations ignore_missing: true - rename: + tag: rename_gcp_metrics_container_instance_count_value_to_gcp_cloudrun_metrics_container_instance_count_89c9fd93 field: gcp.metrics.container.instance_count.value target_field: gcp.cloudrun_metrics.container.instance.count ignore_missing: true - rename: + tag: rename_gcp_metrics_container_max_request_concurrencies_value_to_gcp_cloudrun_metrics_container_max_request_concurrencies_c9418104 field: gcp.metrics.container.max_request_concurrencies.value target_field: gcp.cloudrun_metrics.container.max_request_concurrencies ignore_missing: true - rename: + tag: rename_gcp_metrics_container_memory_allocation_time_value_to_gcp_cloudrun_metrics_container_memory_allocation_time_363f99d0 field: gcp.metrics.container.memory.allocation_time.value target_field: gcp.cloudrun_metrics.container.memory.allocation_time ignore_missing: true - rename: + tag: rename_gcp_metrics_container_memory_utilizations_value_to_gcp_cloudrun_metrics_container_memory_utilizations_2144e944 field: gcp.metrics.container.memory.utilizations.value target_field: gcp.cloudrun_metrics.container.memory.utilizations ignore_missing: true - rename: + tag: rename_gcp_metrics_container_network_received_bytes_count_value_to_gcp_cloudrun_metrics_container_network_received_bytes_aae7d5bf field: gcp.metrics.container.network.received_bytes_count.value target_field: gcp.cloudrun_metrics.container.network.received.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_container_network_sent_bytes_count_value_to_gcp_cloudrun_metrics_container_network_sent_bytes_0862a291 field: gcp.metrics.container.network.sent_bytes_count.value target_field: gcp.cloudrun_metrics.container.network.sent.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_request_count_value_to_gcp_cloudrun_metrics_request_count_2cc26083 field: gcp.metrics.request_count.value target_field: gcp.cloudrun_metrics.request.count ignore_missing: true - rename: + tag: rename_gcp_metrics_request_latencies_value_to_gcp_cloudrun_metrics_request_latencies_8c0ec168 field: gcp.metrics.request_latencies.value target_field: gcp.cloudrun_metrics.request_latencies ignore_missing: true - remove: + tag: remove_ab6e04c7 field: - gcp.metrics ignore_missing: true @@ -61,3 +74,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/gcp/data_stream/cloudsql_mysql/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/cloudsql_mysql/elasticsearch/ingest_pipeline/default.yml index 384f1112da9..ea01d047000 100644 --- a/packages/gcp/data_stream/cloudsql_mysql/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/cloudsql_mysql/elasticsearch/ingest_pipeline/default.yml @@ -2,178 +2,229 @@ description: Pipeline for parsing GCP CloudSQL MySQL metrics. processors: - fingerprint: + tag: fingerprint_5c1d1252 fields: - gcp.labels target_field: gcp.labels_fingerprint ignore_missing: true - drop: + tag: drop_7aad84cd description: Drop if database is not MySQL. if: "ctx?.gcp?.labels?.cloudsql?.name != 'mysql'" - rename: + tag: rename_gcp_metrics_database_auto_failover_request_count_value_to_gcp_cloudsql_mysql_database_auto_failover_request_count_e68a131f field: gcp.metrics.database.auto_failover_request_count.value target_field: gcp.cloudsql_mysql.database.auto_failover_request.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_available_for_failover_value_to_gcp_cloudsql_mysql_database_available_for_failover_3e54db42 field: gcp.metrics.database.available_for_failover.value target_field: gcp.cloudsql_mysql.database.available_for_failover ignore_missing: true - rename: + tag: rename_gcp_metrics_database_cpu_reserved_cores_value_to_gcp_cloudsql_mysql_database_cpu_reserved_cores_count_99514f85 field: gcp.metrics.database.cpu.reserved_cores.value target_field: gcp.cloudsql_mysql.database.cpu.reserved_cores.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_cpu_usage_time_value_to_gcp_cloudsql_mysql_database_cpu_usage_time_sec_a35ddf13 field: gcp.metrics.database.cpu.usage_time.value target_field: gcp.cloudsql_mysql.database.cpu.usage_time.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_database_cpu_utilization_value_to_gcp_cloudsql_mysql_database_cpu_utilization_pct_c9f3e4c7 field: gcp.metrics.database.cpu.utilization.value target_field: gcp.cloudsql_mysql.database.cpu.utilization.pct ignore_missing: true - rename: + tag: rename_gcp_metrics_database_disk_bytes_used_value_to_gcp_cloudsql_mysql_database_disk_bytes_used_bytes_7991d4a7 field: gcp.metrics.database.disk.bytes_used.value target_field: gcp.cloudsql_mysql.database.disk.bytes_used.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_database_disk_quota_value_to_gcp_cloudsql_mysql_database_disk_quota_bytes_8e513699 field: gcp.metrics.database.disk.quota.value target_field: gcp.cloudsql_mysql.database.disk.quota.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_database_disk_read_ops_count_value_to_gcp_cloudsql_mysql_database_disk_read_ops_count_c55e963f field: gcp.metrics.database.disk.read_ops_count.value target_field: gcp.cloudsql_mysql.database.disk.read_ops.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_disk_utilization_value_to_gcp_cloudsql_mysql_database_disk_utilization_pct_c2e1233d field: gcp.metrics.database.disk.utilization.value target_field: gcp.cloudsql_mysql.database.disk.utilization.pct ignore_missing: true - rename: + tag: rename_gcp_metrics_database_disk_write_ops_count_value_to_gcp_cloudsql_mysql_database_disk_write_ops_count_630a8731 field: gcp.metrics.database.disk.write_ops_count.value target_field: gcp.cloudsql_mysql.database.disk.write_ops.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_instance_state_value_to_gcp_cloudsql_mysql_database_instance_state_34c0170a field: gcp.metrics.database.instance_state.value target_field: gcp.cloudsql_mysql.database.instance_state ignore_missing: true - rename: + tag: rename_gcp_metrics_database_memory_quota_value_to_gcp_cloudsql_mysql_database_memory_quota_bytes_4d2dd405 field: gcp.metrics.database.memory.quota.value target_field: gcp.cloudsql_mysql.database.memory.quota.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_database_memory_total_usage_value_to_gcp_cloudsql_mysql_database_memory_total_usage_bytes_feaee859 field: gcp.metrics.database.memory.total_usage.value target_field: gcp.cloudsql_mysql.database.memory.total_usage.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_database_memory_usage_value_to_gcp_cloudsql_mysql_database_memory_usage_bytes_d697e349 field: gcp.metrics.database.memory.usage.value target_field: gcp.cloudsql_mysql.database.memory.usage.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_database_memory_utilization_value_to_gcp_cloudsql_mysql_database_memory_utilization_pct_ab8da371 field: gcp.metrics.database.memory.utilization.value target_field: gcp.cloudsql_mysql.database.memory.utilization.pct ignore_missing: true - rename: + tag: rename_gcp_metrics_database_mysql_innodb_buffer_pool_pages_dirty_value_to_gcp_cloudsql_mysql_database_innodb_buffer_pool_pages_dirty_count_331345b9 field: gcp.metrics.database.mysql.innodb_buffer_pool_pages_dirty.value target_field: gcp.cloudsql_mysql.database.innodb_buffer_pool_pages_dirty.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_mysql_innodb_buffer_pool_pages_free_value_to_gcp_cloudsql_mysql_database_innodb_buffer_pool_pages_free_count_168aa167 field: gcp.metrics.database.mysql.innodb_buffer_pool_pages_free.value target_field: gcp.cloudsql_mysql.database.innodb_buffer_pool_pages_free.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_mysql_innodb_buffer_pool_pages_total_value_to_gcp_cloudsql_mysql_database_innodb_buffer_pool_pages_total_count_d3680a39 field: gcp.metrics.database.mysql.innodb_buffer_pool_pages_total.value target_field: gcp.cloudsql_mysql.database.innodb_buffer_pool_pages_total.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_mysql_innodb_data_fsyncs_value_to_gcp_cloudsql_mysql_database_innodb_data_fsyncs_count_260cbae1 field: gcp.metrics.database.mysql.innodb_data_fsyncs.value target_field: gcp.cloudsql_mysql.database.innodb_data_fsyncs.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_mysql_innodb_os_log_fsyncs_value_to_gcp_cloudsql_mysql_database_innodb_os_log_fsyncs_count_8de5d931 field: gcp.metrics.database.mysql.innodb_os_log_fsyncs.value target_field: gcp.cloudsql_mysql.database.innodb_os_log_fsyncs.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_mysql_innodb_pages_read_value_to_gcp_cloudsql_mysql_database_innodb_pages_read_count_fd9189c7 field: gcp.metrics.database.mysql.innodb_pages_read.value target_field: gcp.cloudsql_mysql.database.innodb_pages_read.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_mysql_innodb_pages_written_value_to_gcp_cloudsql_mysql_database_innodb_pages_written_count_6c398549 field: gcp.metrics.database.mysql.innodb_pages_written.value target_field: gcp.cloudsql_mysql.database.innodb_pages_written.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_mysql_queries_value_to_gcp_cloudsql_mysql_database_queries_count_5621927f field: gcp.metrics.database.mysql.queries.value target_field: gcp.cloudsql_mysql.database.queries.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_mysql_questions_value_to_gcp_cloudsql_mysql_database_questions_count_23a64169 field: gcp.metrics.database.mysql.questions.value target_field: gcp.cloudsql_mysql.database.questions.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_mysql_received_bytes_count_value_to_gcp_cloudsql_mysql_database_received_bytes_count_9820ab85 field: gcp.metrics.database.mysql.received_bytes_count.value target_field: gcp.cloudsql_mysql.database.received_bytes.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_mysql_replication_last_io_errno_value_to_gcp_cloudsql_mysql_database_replication_last_io_errno_a67303c8 field: gcp.metrics.database.mysql.replication.last_io_errno.value target_field: gcp.cloudsql_mysql.database.replication.last_io_errno ignore_missing: true - rename: + tag: rename_gcp_metrics_database_mysql_replication_last_sql_errno_value_to_gcp_cloudsql_mysql_database_replication_last_sql_errno_7f3cb382 field: gcp.metrics.database.mysql.replication.last_sql_errno.value target_field: gcp.cloudsql_mysql.database.replication.last_sql_errno ignore_missing: true - rename: + tag: rename_gcp_metrics_database_mysql_replication_seconds_behind_master_value_to_gcp_cloudsql_mysql_database_replication_seconds_behind_master_sec_ada8d437 field: gcp.metrics.database.mysql.replication.seconds_behind_master.value target_field: gcp.cloudsql_mysql.database.replication.seconds_behind_master.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_database_mysql_replication_slave_io_running_value_to_gcp_cloudsql_mysql_database_replication_slave_io_running_01356886 field: gcp.metrics.database.mysql.replication.slave_io_running.value target_field: gcp.cloudsql_mysql.database.replication.slave_io_running ignore_missing: true - rename: + tag: rename_gcp_metrics_database_mysql_replication_slave_io_running_state_value_to_gcp_cloudsql_mysql_database_replication_slave_io_running_state_6541e5ce field: gcp.metrics.database.mysql.replication.slave_io_running_state.value target_field: gcp.cloudsql_mysql.database.replication.slave_io_running_state ignore_missing: true - rename: + tag: rename_gcp_metrics_database_mysql_replication_slave_sql_running_value_to_gcp_cloudsql_mysql_database_replication_slave_sql_running_acbd1280 field: gcp.metrics.database.mysql.replication.slave_sql_running.value target_field: gcp.cloudsql_mysql.database.replication.slave_sql_running ignore_missing: true - rename: + tag: rename_gcp_metrics_database_mysql_replication_slave_sql_running_state_value_to_gcp_cloudsql_mysql_database_replication_slave_sql_running_state_bfac87ec field: gcp.metrics.database.mysql.replication.slave_sql_running_state.value target_field: gcp.cloudsql_mysql.database.replication.slave_sql_running_state ignore_missing: true - rename: + tag: rename_gcp_metrics_database_mysql_sent_bytes_count_value_to_gcp_cloudsql_mysql_database_sent_bytes_count_b685491d field: gcp.metrics.database.mysql.sent_bytes_count.value target_field: gcp.cloudsql_mysql.database.sent_bytes.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_network_connections_value_to_gcp_cloudsql_mysql_database_network_connections_count_8449db4d field: gcp.metrics.database.network.connections.value target_field: gcp.cloudsql_mysql.database.network.connections.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_network_received_bytes_count_value_to_gcp_cloudsql_mysql_database_network_received_bytes_count_eb388e51 field: gcp.metrics.database.network.received_bytes_count.value target_field: gcp.cloudsql_mysql.database.network.received_bytes.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_network_sent_bytes_count_value_to_gcp_cloudsql_mysql_database_network_sent_bytes_count_459a8d85 field: gcp.metrics.database.network.sent_bytes_count.value target_field: gcp.cloudsql_mysql.database.network.sent_bytes.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_replication_network_lag_value_to_gcp_cloudsql_mysql_database_replication_network_lag_sec_e8e90963 field: gcp.metrics.database.replication.network_lag.value target_field: gcp.cloudsql_mysql.database.replication.network_lag.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_database_replication_replica_lag_value_to_gcp_cloudsql_mysql_database_replication_replica_lag_sec_1df0e847 field: gcp.metrics.database.replication.replica_lag.value target_field: gcp.cloudsql_mysql.database.replication.replica_lag.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_database_up_value_to_gcp_cloudsql_mysql_database_up_0a0a2082 field: gcp.metrics.database.up.value target_field: gcp.cloudsql_mysql.database.up ignore_missing: true - rename: + tag: rename_gcp_metrics_database_uptime_value_to_gcp_cloudsql_mysql_database_uptime_sec_c88ce693 field: gcp.metrics.database.uptime.value target_field: gcp.cloudsql_mysql.database.uptime.sec ignore_missing: true - remove: + tag: remove_ab6e04c7 field: - gcp.metrics ignore_missing: true on_failure: - set: field: error.message - value: '{{ _ingest.on_failure_message }}' \ No newline at end of file + value: '{{ _ingest.on_failure_message }}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/gcp/data_stream/cloudsql_postgresql/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/cloudsql_postgresql/elasticsearch/ingest_pipeline/default.yml index 9e54f097795..f095e69c7ef 100644 --- a/packages/gcp/data_stream/cloudsql_postgresql/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/cloudsql_postgresql/elasticsearch/ingest_pipeline/default.yml @@ -2,202 +2,259 @@ description: Pipeline for parsing GCP CloudSQL PostgreSQL metrics. processors: - fingerprint: + tag: fingerprint_5c1d1252 fields: - gcp.labels target_field: gcp.labels_fingerprint ignore_missing: true - drop: + tag: drop_da6d698c description: Drop if database is not PostgreSQL. if: "ctx?.gcp?.labels?.cloudsql?.name != 'postgres'" - rename: + tag: rename_gcp_metrics_database_auto_failover_request_count_value_to_gcp_cloudsql_postgresql_database_auto_failover_request_count_d772d2a9 field: gcp.metrics.database.auto_failover_request_count.value target_field: gcp.cloudsql_postgresql.database.auto_failover_request.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_available_for_failover_value_to_gcp_cloudsql_postgresql_database_available_for_failover_ea80f218 field: gcp.metrics.database.available_for_failover.value target_field: gcp.cloudsql_postgresql.database.available_for_failover ignore_missing: true - rename: + tag: rename_gcp_metrics_database_cpu_reserved_cores_value_to_gcp_cloudsql_postgresql_database_cpu_reserved_cores_count_75ed20c1 field: gcp.metrics.database.cpu.reserved_cores.value target_field: gcp.cloudsql_postgresql.database.cpu.reserved_cores.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_cpu_usage_time_value_to_gcp_cloudsql_postgresql_database_cpu_usage_time_sec_8c1e7163 field: gcp.metrics.database.cpu.usage_time.value target_field: gcp.cloudsql_postgresql.database.cpu.usage_time.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_database_cpu_utilization_value_to_gcp_cloudsql_postgresql_database_cpu_utilization_pct_e1f544e1 field: gcp.metrics.database.cpu.utilization.value target_field: gcp.cloudsql_postgresql.database.cpu.utilization.pct ignore_missing: true - rename: + tag: rename_gcp_metrics_database_disk_bytes_used_value_to_gcp_cloudsql_postgresql_database_disk_bytes_used_bytes_23759b05 field: gcp.metrics.database.disk.bytes_used.value target_field: gcp.cloudsql_postgresql.database.disk.bytes_used.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_database_disk_quota_value_to_gcp_cloudsql_postgresql_database_disk_quota_bytes_6edf19cd field: gcp.metrics.database.disk.quota.value target_field: gcp.cloudsql_postgresql.database.disk.quota.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_database_disk_read_ops_count_value_to_gcp_cloudsql_postgresql_database_disk_read_ops_count_7ac1bd05 field: gcp.metrics.database.disk.read_ops_count.value target_field: gcp.cloudsql_postgresql.database.disk.read_ops.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_disk_utilization_value_to_gcp_cloudsql_postgresql_database_disk_utilization_pct_b45b5bd1 field: gcp.metrics.database.disk.utilization.value target_field: gcp.cloudsql_postgresql.database.disk.utilization.pct ignore_missing: true - rename: + tag: rename_gcp_metrics_database_disk_write_ops_count_value_to_gcp_cloudsql_postgresql_database_disk_write_ops_count_85c413a5 field: gcp.metrics.database.disk.write_ops_count.value target_field: gcp.cloudsql_postgresql.database.disk.write_ops.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_instance_state_value_to_gcp_cloudsql_postgresql_database_instance_state_36a51e1a field: gcp.metrics.database.instance_state.value target_field: gcp.cloudsql_postgresql.database.instance_state ignore_missing: true - rename: + tag: rename_gcp_metrics_database_memory_quota_value_to_gcp_cloudsql_postgresql_database_memory_quota_bytes_e6809a95 field: gcp.metrics.database.memory.quota.value target_field: gcp.cloudsql_postgresql.database.memory.quota.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_database_memory_total_usage_value_to_gcp_cloudsql_postgresql_database_memory_total_usage_bytes_cdd3228d field: gcp.metrics.database.memory.total_usage.value target_field: gcp.cloudsql_postgresql.database.memory.total_usage.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_database_memory_usage_value_to_gcp_cloudsql_postgresql_database_memory_usage_bytes_615ea0db field: gcp.metrics.database.memory.usage.value target_field: gcp.cloudsql_postgresql.database.memory.usage.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_database_memory_utilization_value_to_gcp_cloudsql_postgresql_database_memory_utilization_pct_074c9e21 field: gcp.metrics.database.memory.utilization.value target_field: gcp.cloudsql_postgresql.database.memory.utilization.pct ignore_missing: true - rename: + tag: rename_gcp_metrics_database_network_connections_value_to_gcp_cloudsql_postgresql_database_network_connections_count_9d1e0459 field: gcp.metrics.database.network.connections.value target_field: gcp.cloudsql_postgresql.database.network.connections.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_network_received_bytes_count_value_to_gcp_cloudsql_postgresql_database_network_received_bytes_count_c82ff51d field: gcp.metrics.database.network.received_bytes_count.value target_field: gcp.cloudsql_postgresql.database.network.received_bytes.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_network_sent_bytes_count_value_to_gcp_cloudsql_postgresql_database_network_sent_bytes_count_86a0ade3 field: gcp.metrics.database.network.sent_bytes_count.value target_field: gcp.cloudsql_postgresql.database.network.sent_bytes.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_insights_aggregate_execution_time_value_to_gcp_cloudsql_postgresql_database_insights_aggregate_execution_time_ee8d5914 field: gcp.metrics.database.postgresql.insights.aggregate.execution_time.value target_field: gcp.cloudsql_postgresql.database.insights.aggregate.execution_time ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_insights_aggregate_io_time_value_to_gcp_cloudsql_postgresql_database_insights_aggregate_io_time_c6ea17ce field: gcp.metrics.database.postgresql.insights.aggregate.io_time.value target_field: gcp.cloudsql_postgresql.database.insights.aggregate.io_time ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_insights_aggregate_latencies_value_to_gcp_cloudsql_postgresql_database_insights_aggregate_latencies_9f28a72a field: gcp.metrics.database.postgresql.insights.aggregate.latencies.value target_field: gcp.cloudsql_postgresql.database.insights.aggregate.latencies ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_insights_aggregate_lock_time_value_to_gcp_cloudsql_postgresql_database_insights_aggregate_lock_time_9b8c26a0 field: gcp.metrics.database.postgresql.insights.aggregate.lock_time.value target_field: gcp.cloudsql_postgresql.database.insights.aggregate.lock_time ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_insights_aggregate_row_count_value_to_gcp_cloudsql_postgresql_database_insights_aggregate_row_count_72f2fc31 field: gcp.metrics.database.postgresql.insights.aggregate.row_count.value target_field: gcp.cloudsql_postgresql.database.insights.aggregate.row.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_insights_aggregate_shared_blk_access_count_value_to_gcp_cloudsql_postgresql_database_insights_aggregate_shared_blk_access_count_607b79bd field: gcp.metrics.database.postgresql.insights.aggregate.shared_blk_access_count.value target_field: gcp.cloudsql_postgresql.database.insights.aggregate.shared_blk_access.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_insights_perquery_execution_time_value_to_gcp_cloudsql_postgresql_database_insights_perquery_execution_time_6bef3bfa field: gcp.metrics.database.postgresql.insights.perquery.execution_time.value target_field: gcp.cloudsql_postgresql.database.insights.perquery.execution_time ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_insights_perquery_io_time_value_to_gcp_cloudsql_postgresql_database_insights_perquery_io_time_0300658c field: gcp.metrics.database.postgresql.insights.perquery.io_time.value target_field: gcp.cloudsql_postgresql.database.insights.perquery.io_time ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_insights_perquery_latencies_value_to_gcp_cloudsql_postgresql_database_insights_perquery_latencies_917214d4 field: gcp.metrics.database.postgresql.insights.perquery.latencies.value target_field: gcp.cloudsql_postgresql.database.insights.perquery.latencies ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_insights_perquery_lock_time_value_to_gcp_cloudsql_postgresql_database_insights_perquery_lock_time_9e8110b4 field: gcp.metrics.database.postgresql.insights.perquery.lock_time.value target_field: gcp.cloudsql_postgresql.database.insights.perquery.lock_time ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_insights_perquery_row_count_value_to_gcp_cloudsql_postgresql_database_insights_perquery_row_count_18558657 field: gcp.metrics.database.postgresql.insights.perquery.row_count.value target_field: gcp.cloudsql_postgresql.database.insights.perquery.row.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_insights_perquery_shared_blk_access_count_value_to_gcp_cloudsql_postgresql_database_insights_perquery_shared_blk_access_count_12e5183b field: gcp.metrics.database.postgresql.insights.perquery.shared_blk_access_count.value target_field: gcp.cloudsql_postgresql.database.insights.perquery.shared_blk_access.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_insights_pertag_execution_time_value_to_gcp_cloudsql_postgresql_database_insights_pertag_execution_time_a0bb20fe field: gcp.metrics.database.postgresql.insights.pertag.execution_time.value target_field: gcp.cloudsql_postgresql.database.insights.pertag.execution_time ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_insights_pertag_io_time_value_to_gcp_cloudsql_postgresql_database_insights_pertag_io_time_bbb7eff0 field: gcp.metrics.database.postgresql.insights.pertag.io_time.value target_field: gcp.cloudsql_postgresql.database.insights.pertag.io_time ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_insights_pertag_latencies_value_to_gcp_cloudsql_postgresql_database_insights_pertag_latencies_dc5ea278 field: gcp.metrics.database.postgresql.insights.pertag.latencies.value target_field: gcp.cloudsql_postgresql.database.insights.pertag.latencies ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_insights_pertag_lock_time_value_to_gcp_cloudsql_postgresql_database_insights_pertag_lock_time_efe18d14 field: gcp.metrics.database.postgresql.insights.pertag.lock_time.value target_field: gcp.cloudsql_postgresql.database.insights.pertag.lock_time ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_insights_pertag_row_count_value_to_gcp_cloudsql_postgresql_database_insights_pertag_row_count_c6b8b66b field: gcp.metrics.database.postgresql.insights.pertag.row_count.value target_field: gcp.cloudsql_postgresql.database.insights.pertag.row.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_insights_pertag_shared_blk_access_count_value_to_gcp_cloudsql_postgresql_database_insights_pertag_shared_blk_access_count_08c3cc0f field: gcp.metrics.database.postgresql.insights.pertag.shared_blk_access_count.value target_field: gcp.cloudsql_postgresql.database.insights.pertag.shared_blk_access.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_num_backends_value_to_gcp_cloudsql_postgresql_database_num_backends_count_da8777e5 field: gcp.metrics.database.postgresql.num_backends.value target_field: gcp.cloudsql_postgresql.database.num_backends.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_replication_replica_byte_lag_value_to_gcp_cloudsql_postgresql_database_replication_replica_byte_lag_bytes_7e9f7411 field: gcp.metrics.database.postgresql.replication.replica_byte_lag.value target_field: gcp.cloudsql_postgresql.database.replication.replica_byte_lag.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_transaction_count_value_to_gcp_cloudsql_postgresql_database_transaction_count_9f3216af field: gcp.metrics.database.postgresql.transaction_count.value target_field: gcp.cloudsql_postgresql.database.transaction.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_transaction_id_count_value_to_gcp_cloudsql_postgresql_database_transaction_id_count_1ca5d9b5 field: gcp.metrics.database.postgresql.transaction_id_count.value target_field: gcp.cloudsql_postgresql.database.transaction_id.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_transaction_id_utilization_value_to_gcp_cloudsql_postgresql_database_transaction_id_utilization_pct_7cbfec9f field: gcp.metrics.database.postgresql.transaction_id_utilization.value target_field: gcp.cloudsql_postgresql.database.transaction_id_utilization.pct ignore_missing: true - rename: + tag: rename_gcp_metrics_database_postgresql_vacuum_oldest_transaction_age_value_to_gcp_cloudsql_postgresql_database_vacuum_oldest_transaction_age_5e4db30c field: gcp.metrics.database.postgresql.vacuum.oldest_transaction_age.value target_field: gcp.cloudsql_postgresql.database.vacuum.oldest_transaction_age ignore_missing: true - rename: + tag: rename_gcp_metrics_database_replication_network_lag_value_to_gcp_cloudsql_postgresql_database_replication_network_lag_sec_6f6348cf field: gcp.metrics.database.replication.network_lag.value target_field: gcp.cloudsql_postgresql.database.replication.network_lag.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_database_replication_replica_lag_value_to_gcp_cloudsql_postgresql_database_replication_replica_lag_sec_edebe797 field: gcp.metrics.database.replication.replica_lag.value target_field: gcp.cloudsql_postgresql.database.replication.replica_lag.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_database_up_value_to_gcp_cloudsql_postgresql_database_up_3a38dc92 field: gcp.metrics.database.up.value target_field: gcp.cloudsql_postgresql.database.up ignore_missing: true - rename: + tag: rename_gcp_metrics_database_uptime_value_to_gcp_cloudsql_postgresql_database_uptime_sec_8b9498bd field: gcp.metrics.database.uptime.value target_field: gcp.cloudsql_postgresql.database.uptime.sec ignore_missing: true - remove: + tag: remove_ab6e04c7 field: - gcp.metrics ignore_missing: true on_failure: - set: field: error.message - value: '{{ _ingest.on_failure_message }}' \ No newline at end of file + value: '{{ _ingest.on_failure_message }}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/gcp/data_stream/cloudsql_sqlserver/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/cloudsql_sqlserver/elasticsearch/ingest_pipeline/default.yml index 92a2cd452d2..0f69c514563 100644 --- a/packages/gcp/data_stream/cloudsql_sqlserver/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/cloudsql_sqlserver/elasticsearch/ingest_pipeline/default.yml @@ -2,114 +2,149 @@ description: Pipeline for parsing GCP CloudSQL SQL Server metrics. processors: - fingerprint: + tag: fingerprint_5c1d1252 fields: - gcp.labels target_field: gcp.labels_fingerprint ignore_missing: true - drop: + tag: drop_1adbe437 description: Drop if database is not SQLServer. if: "ctx?.gcp?.labels?.cloudsql?.name != 'sqlserver'" - rename: + tag: rename_gcp_metrics_database_auto_failover_request_count_value_to_gcp_cloudsql_sqlserver_database_auto_failover_request_count_d19ae166 field: gcp.metrics.database.auto_failover_request_count.value target_field: gcp.cloudsql_sqlserver.database.auto_failover_request.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_available_for_failover_value_to_gcp_cloudsql_sqlserver_database_available_for_failover_de7ed251 field: gcp.metrics.database.available_for_failover.value target_field: gcp.cloudsql_sqlserver.database.available_for_failover ignore_missing: true - rename: + tag: rename_gcp_metrics_database_cpu_reserved_cores_value_to_gcp_cloudsql_sqlserver_database_cpu_reserved_cores_count_dc3c780e field: gcp.metrics.database.cpu.reserved_cores.value target_field: gcp.cloudsql_sqlserver.database.cpu.reserved_cores.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_cpu_usage_time_value_to_gcp_cloudsql_sqlserver_database_cpu_usage_time_sec_bc8a8248 field: gcp.metrics.database.cpu.usage_time.value target_field: gcp.cloudsql_sqlserver.database.cpu.usage_time.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_database_cpu_utilization_value_to_gcp_cloudsql_sqlserver_database_cpu_utilization_pct_54876bda field: gcp.metrics.database.cpu.utilization.value target_field: gcp.cloudsql_sqlserver.database.cpu.utilization.pct ignore_missing: true - rename: + tag: rename_gcp_metrics_database_disk_bytes_used_value_to_gcp_cloudsql_sqlserver_database_disk_bytes_used_bytes_5ae242e2 field: gcp.metrics.database.disk.bytes_used.value target_field: gcp.cloudsql_sqlserver.database.disk.bytes_used.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_database_disk_quota_value_to_gcp_cloudsql_sqlserver_database_disk_quota_bytes_6bffa332 field: gcp.metrics.database.disk.quota.value target_field: gcp.cloudsql_sqlserver.database.disk.quota.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_database_disk_read_ops_count_value_to_gcp_cloudsql_sqlserver_database_disk_read_ops_count_c10719ea field: gcp.metrics.database.disk.read_ops_count.value target_field: gcp.cloudsql_sqlserver.database.disk.read_ops.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_disk_utilization_value_to_gcp_cloudsql_sqlserver_database_disk_utilization_pct_e519a31a field: gcp.metrics.database.disk.utilization.value target_field: gcp.cloudsql_sqlserver.database.disk.utilization.pct ignore_missing: true - rename: + tag: rename_gcp_metrics_database_disk_write_ops_count_value_to_gcp_cloudsql_sqlserver_database_disk_write_ops_count_76cb59fa field: gcp.metrics.database.disk.write_ops_count.value target_field: gcp.cloudsql_sqlserver.database.disk.write_ops.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_instance_state_value_to_gcp_cloudsql_sqlserver_database_instance_state_387b6d95 field: gcp.metrics.database.instance_state.value target_field: gcp.cloudsql_sqlserver.database.instance_state ignore_missing: true - rename: + tag: rename_gcp_metrics_database_memory_quota_value_to_gcp_cloudsql_sqlserver_database_memory_quota_bytes_079c2632 field: gcp.metrics.database.memory.quota.value target_field: gcp.cloudsql_sqlserver.database.memory.quota.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_database_memory_total_usage_value_to_gcp_cloudsql_sqlserver_database_memory_total_usage_bytes_358b9f92 field: gcp.metrics.database.memory.total_usage.value target_field: gcp.cloudsql_sqlserver.database.memory.total_usage.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_database_memory_usage_value_to_gcp_cloudsql_sqlserver_database_memory_usage_bytes_42d53e7a field: gcp.metrics.database.memory.usage.value target_field: gcp.cloudsql_sqlserver.database.memory.usage.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_database_memory_utilization_value_to_gcp_cloudsql_sqlserver_database_memory_utilization_pct_ae6b1562 field: gcp.metrics.database.memory.utilization.value target_field: gcp.cloudsql_sqlserver.database.memory.utilization.pct ignore_missing: true - rename: + tag: rename_gcp_metrics_database_network_connections_value_to_gcp_cloudsql_sqlserver_database_network_connections_count_5ec3bca4 field: gcp.metrics.database.network.connections.value target_field: gcp.cloudsql_sqlserver.database.network.connections.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_network_received_bytes_count_value_to_gcp_cloudsql_sqlserver_database_network_received_bytes_count_345653aa field: gcp.metrics.database.network.received_bytes_count.value target_field: gcp.cloudsql_sqlserver.database.network.received_bytes.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_network_sent_bytes_count_value_to_gcp_cloudsql_sqlserver_database_network_sent_bytes_count_41408a66 field: gcp.metrics.database.network.sent_bytes_count.value target_field: gcp.cloudsql_sqlserver.database.network.sent_bytes.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_replication_network_lag_value_to_gcp_cloudsql_sqlserver_database_replication_network_lag_sec_3b18e6ee field: gcp.metrics.database.replication.network_lag.value target_field: gcp.cloudsql_sqlserver.database.replication.network_lag.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_database_replication_replica_lag_value_to_gcp_cloudsql_sqlserver_database_replication_replica_lag_sec_58c5607e field: gcp.metrics.database.replication.replica_lag.value target_field: gcp.cloudsql_sqlserver.database.replication.replica_lag.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_database_sqlserver_audits_size_value_to_gcp_cloudsql_sqlserver_database_audits_size_bytes_ef845a2f field: gcp.metrics.database.sqlserver.audits_size.value target_field: gcp.cloudsql_sqlserver.database.audits_size.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_database_sqlserver_audits_upload_count_value_to_gcp_cloudsql_sqlserver_database_audits_upload_count_c59ecd1f field: gcp.metrics.database.sqlserver.audits_upload_count.value target_field: gcp.cloudsql_sqlserver.database.audits_upload.count ignore_missing: true - rename: + tag: rename_gcp_metrics_database_up_value_to_gcp_cloudsql_sqlserver_database_up_01fa7721 field: gcp.metrics.database.up.value target_field: gcp.cloudsql_sqlserver.database.up ignore_missing: true - rename: + tag: rename_gcp_metrics_database_uptime_value_to_gcp_cloudsql_sqlserver_database_uptime_sec_6f91863c field: gcp.metrics.database.uptime.value target_field: gcp.cloudsql_sqlserver.database.uptime.sec ignore_missing: true - remove: + tag: remove_ab6e04c7 field: - gcp.metrics ignore_missing: true on_failure: - set: field: error.message - value: '{{ _ingest.on_failure_message }}' \ No newline at end of file + value: '{{ _ingest.on_failure_message }}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/gcp/data_stream/compute/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/compute/elasticsearch/ingest_pipeline/default.yml index e1bd1dc2b25..272bac03428 100644 --- a/packages/gcp/data_stream/compute/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/compute/elasticsearch/ingest_pipeline/default.yml @@ -2,87 +2,108 @@ description: Pipeline for parsing GCP Compute metrics. processors: - fingerprint: + tag: fingerprint_5c1d1252 fields: - gcp.labels target_field: gcp.labels_fingerprint ignore_missing: true - rename: + tag: rename_gcp_metrics_firewall_dropped_bytes_to_gcp_compute_firewall_dropped_bytes_bf5202b8 field: gcp.metrics.firewall.dropped.bytes target_field: gcp.compute.firewall.dropped.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_firewall_dropped_packets_count_value_to_gcp_compute_firewall_dropped_packets_count_value_ff2d8334 field: gcp.metrics.firewall.dropped_packets_count.value target_field: gcp.compute.firewall.dropped_packets_count.value ignore_missing: true - rename: + tag: rename_gcp_metrics_instance_cpu_reserved_cores_value_to_gcp_compute_instance_cpu_reserved_cores_value_782b480a field: gcp.metrics.instance.cpu.reserved_cores.value target_field: gcp.compute.instance.cpu.reserved_cores.value ignore_missing: true - rename: + tag: rename_gcp_metrics_instance_cpu_usage_time_sec_to_gcp_compute_instance_cpu_usage_time_sec_e1f1f736 field: gcp.metrics.instance.cpu.usage_time.sec target_field: gcp.compute.instance.cpu.usage_time.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_instance_cpu_usage_pct_to_gcp_compute_instance_cpu_usage_pct_080854f4 field: gcp.metrics.instance.cpu.usage.pct target_field: gcp.compute.instance.cpu.usage.pct ignore_missing: true - rename: + tag: rename_gcp_metrics_instance_disk_read_bytes_to_gcp_compute_instance_disk_read_bytes_6332c2e8 field: gcp.metrics.instance.disk.read.bytes target_field: gcp.compute.instance.disk.read.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_instance_disk_read_ops_count_value_to_gcp_compute_instance_disk_read_ops_count_value_2db757e8 field: gcp.metrics.instance.disk.read_ops_count.value target_field: gcp.compute.instance.disk.read_ops_count.value ignore_missing: true - rename: + tag: rename_gcp_metrics_instance_disk_write_bytes_to_gcp_compute_instance_disk_write_bytes_45faa37c field: gcp.metrics.instance.disk.write.bytes target_field: gcp.compute.instance.disk.write.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_instance_disk_write_ops_count_value_to_gcp_compute_instance_disk_write_ops_count_value_067284de field: gcp.metrics.instance.disk.write_ops_count.value target_field: gcp.compute.instance.disk.write_ops_count.value ignore_missing: true - rename: + tag: rename_gcp_metrics_instance_memory_balloon_ram_size_value_to_gcp_compute_instance_memory_balloon_ram_size_value_d61ef8c8 field: gcp.metrics.instance.memory.balloon.ram_size.value target_field: gcp.compute.instance.memory.balloon.ram_size.value ignore_missing: true - rename: + tag: rename_gcp_metrics_instance_memory_balloon_ram_used_value_to_gcp_compute_instance_memory_balloon_ram_used_value_0426f174 field: gcp.metrics.instance.memory.balloon.ram_used.value target_field: gcp.compute.instance.memory.balloon.ram_used.value ignore_missing: true - rename: + tag: rename_gcp_metrics_instance_memory_balloon_swap_in_bytes_to_gcp_compute_instance_memory_balloon_swap_in_bytes_dbb886aa field: gcp.metrics.instance.memory.balloon.swap_in.bytes target_field: gcp.compute.instance.memory.balloon.swap_in.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_instance_memory_balloon_swap_out_bytes_to_gcp_compute_instance_memory_balloon_swap_out_bytes_bf7e9868 field: gcp.metrics.instance.memory.balloon.swap_out.bytes target_field: gcp.compute.instance.memory.balloon.swap_out.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_instance_network_ingress_bytes_to_gcp_compute_instance_network_ingress_bytes_1a0a1ec4 field: gcp.metrics.instance.network.ingress.bytes target_field: gcp.compute.instance.network.ingress.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_instance_network_ingress_packets_count_to_gcp_compute_instance_network_ingress_packets_count_598bf998 field: gcp.metrics.instance.network.ingress.packets.count target_field: gcp.compute.instance.network.ingress.packets.count ignore_missing: true - rename: + tag: rename_gcp_metrics_instance_network_egress_bytes_to_gcp_compute_instance_network_egress_bytes_c6529ef2 field: gcp.metrics.instance.network.egress.bytes target_field: gcp.compute.instance.network.egress.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_instance_network_egress_packets_count_to_gcp_compute_instance_network_egress_packets_count_55c0ba5c field: gcp.metrics.instance.network.egress.packets.count target_field: gcp.compute.instance.network.egress.packets.count ignore_missing: true - rename: + tag: rename_gcp_metrics_instance_uptime_sec_to_gcp_compute_instance_uptime_sec_456b8270 field: gcp.metrics.instance.uptime.sec target_field: gcp.compute.instance.uptime.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_instance_uptime_total_sec_to_gcp_compute_instance_uptime_total_sec_10fa98b6 field: gcp.metrics.instance.uptime_total.sec target_field: gcp.compute.instance.uptime_total.sec ignore_missing: true - remove: + tag: remove_ab6e04c7 field: - gcp.metrics ignore_missing: true @@ -93,3 +114,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/gcp/data_stream/dataproc/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/dataproc/elasticsearch/ingest_pipeline/default.yml index bcc011d8922..7e67efba4a8 100644 --- a/packages/gcp/data_stream/dataproc/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/dataproc/elasticsearch/ingest_pipeline/default.yml @@ -2,103 +2,127 @@ description: Pipeline for parsing GCP Dataproc metrics. processors: - fingerprint: + tag: fingerprint_5c1d1252 fields: - gcp.labels target_field: gcp.labels_fingerprint ignore_missing: true - rename: + tag: rename_gcp_metrics_batch_spark_executors_count_to_gcp_dataproc_batch_spark_executors_count_8322ebf1 field: gcp.metrics.batch.spark.executors.count target_field: gcp.dataproc.batch.spark.executors.count ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_hdfs_datanodes_count_to_gcp_dataproc_cluster_hdfs_datanodes_count_a279818d field: gcp.metrics.cluster.hdfs.datanodes.count target_field: gcp.dataproc.cluster.hdfs.datanodes.count ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_hdfs_storage_capacity_value_to_gcp_dataproc_cluster_hdfs_storage_capacity_value_8c6ba60d field: gcp.metrics.cluster.hdfs.storage_capacity.value target_field: gcp.dataproc.cluster.hdfs.storage_capacity.value ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_hdfs_storage_utilization_value_to_gcp_dataproc_cluster_hdfs_storage_utilization_value_12b8e11f field: gcp.metrics.cluster.hdfs.storage_utilization.value target_field: gcp.dataproc.cluster.hdfs.storage_utilization.value ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_hdfs_unhealthy_blocks_count_to_gcp_dataproc_cluster_hdfs_unhealthy_blocks_count_698842b1 field: gcp.metrics.cluster.hdfs.unhealthy_blocks.count target_field: gcp.dataproc.cluster.hdfs.unhealthy_blocks.count ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_job_failed_count_to_gcp_dataproc_cluster_job_failed_count_1ef788c1 field: gcp.metrics.cluster.job.failed.count target_field: gcp.dataproc.cluster.job.failed.count ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_job_running_count_to_gcp_dataproc_cluster_job_running_count_6eab8ff1 field: gcp.metrics.cluster.job.running.count target_field: gcp.dataproc.cluster.job.running.count ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_job_submitted_count_to_gcp_dataproc_cluster_job_submitted_count_43515c51 field: gcp.metrics.cluster.job.submitted.count target_field: gcp.dataproc.cluster.job.submitted.count ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_operation_failed_count_to_gcp_dataproc_cluster_operation_failed_count_6fd38b8d field: gcp.metrics.cluster.operation.failed.count target_field: gcp.dataproc.cluster.operation.failed.count ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_operation_running_count_to_gcp_dataproc_cluster_operation_running_count_b205ba09 field: gcp.metrics.cluster.operation.running.count target_field: gcp.dataproc.cluster.operation.running.count ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_operation_submitted_count_to_gcp_dataproc_cluster_operation_submitted_count_8fe718e1 field: gcp.metrics.cluster.operation.submitted.count target_field: gcp.dataproc.cluster.operation.submitted.count ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_yarn_allocated_memory_percentage_value_to_gcp_dataproc_cluster_yarn_allocated_memory_percentage_value_59211379 field: gcp.metrics.cluster.yarn.allocated_memory_percentage.value target_field: gcp.dataproc.cluster.yarn.allocated_memory_percentage.value ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_yarn_apps_count_to_gcp_dataproc_cluster_yarn_apps_count_7f8a9d51 field: gcp.metrics.cluster.yarn.apps.count target_field: gcp.dataproc.cluster.yarn.apps.count ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_yarn_containers_count_to_gcp_dataproc_cluster_yarn_containers_count_a4211c85 field: gcp.metrics.cluster.yarn.containers.count target_field: gcp.dataproc.cluster.yarn.containers.count ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_yarn_memory_size_value_to_gcp_dataproc_cluster_yarn_memory_size_value_95277d37 field: gcp.metrics.cluster.yarn.memory_size.value target_field: gcp.dataproc.cluster.yarn.memory_size.value ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_yarn_nodemanagers_count_to_gcp_dataproc_cluster_yarn_nodemanagers_count_ed56e85d field: gcp.metrics.cluster.yarn.nodemanagers.count target_field: gcp.dataproc.cluster.yarn.nodemanagers.count ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_yarn_pending_memory_size_value_to_gcp_dataproc_cluster_yarn_pending_memory_size_value_f432c14f field: gcp.metrics.cluster.yarn.pending_memory_size.value target_field: gcp.dataproc.cluster.yarn.pending_memory_size.value ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_yarn_virtual_cores_count_to_gcp_dataproc_cluster_yarn_virtual_cores_count_5318d1fd field: gcp.metrics.cluster.yarn.virtual_cores.count target_field: gcp.dataproc.cluster.yarn.virtual_cores.count ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_job_completion_time_value_to_gcp_dataproc_cluster_job_completion_time_value_f7dac489 field: gcp.metrics.cluster.job.completion_time.value target_field: gcp.dataproc.cluster.job.completion_time.value ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_job_duration_value_to_gcp_dataproc_cluster_job_duration_value_ce3a4ecb field: gcp.metrics.cluster.job.duration.value target_field: gcp.dataproc.cluster.job.duration.value ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_operation_completion_time_value_to_gcp_dataproc_cluster_operation_completion_time_value_dacdd3ed field: gcp.metrics.cluster.operation.completion_time.value target_field: gcp.dataproc.cluster.operation.completion_time.value ignore_missing: true - rename: + tag: rename_gcp_metrics_cluster_operation_duration_value_to_gcp_dataproc_cluster_operation_duration_value_645d5c13 field: gcp.metrics.cluster.operation.duration.value target_field: gcp.dataproc.cluster.operation.duration.value ignore_missing: true - remove: + tag: remove_ab6e04c7 ignore_missing: true field: - gcp.metrics - + on_failure: - set: field: event.kind @@ -106,3 +130,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/gcp/data_stream/dns/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/dns/elasticsearch/ingest_pipeline/default.yml index 1cbf60c27a3..f2c15214c39 100644 --- a/packages/gcp/data_stream/dns/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/dns/elasticsearch/ingest_pipeline/default.yml @@ -2,132 +2,164 @@ description: Pipeline for Google Cloud DNS logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true - if: 'ctx.event?.original == null' + if: 'ctx.event?.original == null' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_category_dbab8a4e field: event.category value: ["network"] - set: + tag: set_event_action_67c39ee2 field: event.action value: dns-query - set: + tag: set_cloud_provider_62c5e42e field: cloud.provider value: gcp - date: + tag: date_json_timestamp_e111e976 field: json.timestamp timezone: UTC formats: - ISO8601 - set: + tag: set_log_logger_01230fab field: log.logger copy_from: json.logName ignore_failure: true - set: + tag: set_log_level_d2cd6285 field: log.level copy_from: json.severity ignore_failure: true - set: + tag: set_event_id_efd37a25 field: event.id copy_from: json.insertId ignore_empty_value: true ignore_failure: true - - convert: + - convert: + tag: convert_json_resource_labels_project_id_to_cloud_project_id_390fb252 field: json.resource.labels.project_id target_field: cloud.project.id type: string ignore_missing: true ignore_failure: true - - convert: + - convert: + tag: convert_json_resource_labels_location_to_cloud_region_f1d4aea1 field: json.resource.labels.location target_field: cloud.region type: string ignore_failure: true - set: + tag: set_gcp_dns_auth_answer_c0273364 field: gcp.dns.auth_answer copy_from: json.jsonPayload.authAnswer ignore_failure: true - set: + tag: set_gcp_dns_destination_ip_7b975d80 field: gcp.dns.destination_ip copy_from: json.jsonPayload.destinationIP ignore_failure: true - set: + tag: set_destination_address_6248e853 field: destination.address copy_from: gcp.dns.destination_ip ignore_failure: true - convert: + tag: convert_gcp_dns_destination_ip_to_destination_ip_8202a074 field: gcp.dns.destination_ip target_field: destination.ip type: ip ignore_failure: true - set: + tag: set_gcp_dns_egress_error_07d3554c field: gcp.dns.egress_error copy_from: json.jsonPayload.egressError ignore_failure: true - set: + tag: set_gcp_dns_protocol_4a5570bf field: gcp.dns.protocol copy_from: json.jsonPayload.protocol ignore_failure: true - set: + tag: set_network_transport_0322102e field: network.transport copy_from: gcp.dns.protocol ignore_failure: true - lowercase: + tag: lowercase_network_transport_bc8c1c12 field: network.transport ignore_missing: true - set: + tag: set_network_iana_number_c83eed1a field: network.iana_number value: '6' if: ctx.network?.transport == "tcp" - set: + tag: set_network_iana_number_c552b054 field: network.iana_number value: '17' if: ctx.network?.transport == "udp" - set: + tag: set_network_protocol_96a840d1 field: network.protocol value: dns - set: + tag: set_gcp_dns_query_name_1ed212b0 field: gcp.dns.query_name copy_from: json.jsonPayload.queryName ignore_failure: true - set: + tag: set_dns_question_name_834cf51f field: dns.question.name copy_from: gcp.dns.query_name ignore_failure: true - gsub: + tag: gsub_dns_question_name_5c33b959 field: dns.question.name pattern: "[.]$" replacement: "" ignore_failure: true - registered_domain: + tag: registered_domain_dns_question_name_to_dns_question_c90ad4ff field: dns.question.name target_field: dns.question - remove: + tag: remove_dns_question_domain_3403bb7d field: dns.question.domain ignore_missing: true - set: + tag: set_gcp_dns_query_type_569889b2 field: gcp.dns.query_type copy_from: json.jsonPayload.queryType ignore_failure: true - set: + tag: set_dns_question_type_b79fcafb field: dns.question.type copy_from: gcp.dns.query_type ignore_failure: true - set: + tag: set_gcp_dns_rdata_1320d1c1 field: gcp.dns.rdata copy_from: json.jsonPayload.rdata ignore_failure: true -### Internal DNS query parsing + ### Internal DNS query parsing - script: if: ctx.gcp?.dns?.rdata != null && ctx.gcp.dns.rdata != "" lang: painless @@ -174,8 +206,9 @@ processors: ]); } ctx.dns.answers = dns_answers; -### External DNS query parsing + ### External DNS query parsing - script: + tag: script_223ceb8b lang: painless ignore_failure: true description: This script processes the Public DNS RData into `dns.answers`. @@ -209,6 +242,7 @@ processors: } ctx.dns.answers = answers; - script: + tag: script_81999a8f lang: painless ignore_failure: true if: ctx.dns?.answers != null && ctx.dns?.answers instanceof List @@ -245,108 +279,133 @@ processors: } } - set: + tag: set_gcp_dns_response_code_3c9fd546 field: gcp.dns.response_code copy_from: json.jsonPayload.responseCode ignore_failure: true - set: + tag: set_dns_response_code_e170bbe2 field: dns.response_code copy_from: gcp.dns.response_code ignore_failure: true - set: + tag: set_event_outcome_6c09e6cc field: event.outcome value: success if: ctx?.gcp?.dns?.response_code != null && ctx?.gcp?.dns?.response_code == "NOERROR" - set: + tag: set_event_outcome_ce1e8037 field: event.outcome value: failure if: ctx?.gcp?.dns?.response_code != null && ctx?.gcp?.dns?.response_code != "NOERROR" - set: + tag: set_gcp_dns_server_latency_4141f4b0 field: gcp.dns.server_latency copy_from: json.jsonPayload.serverLatency ignore_failure: true - set: + tag: set_gcp_dns_source_ip_d2028402 field: gcp.dns.source_ip copy_from: json.jsonPayload.sourceIP ignore_failure: true - set: + tag: set_source_address_63abf5c7 field: source.address copy_from: gcp.dns.source_ip ignore_failure: true - convert: + tag: convert_gcp_dns_source_ip_to_source_ip_cfa9c148 field: gcp.dns.source_ip target_field: source.ip type: ip ignore_failure: true - set: + tag: set_gcp_dns_source_network_dc7a8a50 field: gcp.dns.source_network copy_from: json.jsonPayload.sourceNetwork ignore_failure: true - set: + tag: set_gcp_dns_vm_instance_id_b7c72bac field: gcp.dns.vm_instance_id copy_from: json.jsonPayload.vmInstanceIdString ignore_failure: true - set: + tag: set_cloud_instance_id_c730509b field: cloud.instance.id copy_from: gcp.dns.vm_instance_id ignore_failure: true - set: + tag: set_gcp_dns_vm_instance_name_9c9a44a5 field: gcp.dns.vm_instance_name copy_from: json.jsonPayload.vmInstanceName ignore_failure: true - set: + tag: set_cloud_instance_name_0eeff2a7 field: cloud.instance.name copy_from: gcp.dns.vm_instance_name ignore_failure: true - gsub: + tag: gsub_cloud_instance_name_1797899e field: cloud.instance.name pattern: "^.*[.]" replacement: "" ignore_failure: true - set: + tag: set_gcp_dns_vm_project_id_dbb74a83 field: gcp.dns.vm_project_id copy_from: json.jsonPayload.vmProjectId ignore_failure: true - set: + tag: set_gcp_dns_vm_zone_name_8f171271 field: gcp.dns.vm_zone_name copy_from: json.jsonPayload.vmZoneName ignore_failure: true - set: + tag: set_cloud_availability_zone_e0244264 field: cloud.availability_zone copy_from: gcp.dns.vm_zone_name ignore_failure: true - set: + tag: set_gcp_dns_target_type_085129ae field: gcp.dns.target_type copy_from: json.resource.labels.target_type ignore_failure: true - set: + tag: set_gcp_dns_source_type_2a595442 field: gcp.dns.source_type copy_from: json.resource.labels.source_type ignore_failure: true - append: + tag: append_related_ip_d7aecc5f field: related.ip value: "{{source.ip}}" allow_duplicates: false if: ctx.source?.ip != null - append: + tag: append_related_ip_1164e6c1 field: related.ip value: "{{destination.ip}}" allow_duplicates: false if: ctx.destination?.ip != null - append: + tag: append_related_hosts_adfc73a4 field: related.hosts value: "{{dns.question.name}}" allow_duplicates: false if: ctx.dns?.question?.name != null - rename: + tag: rename_json_to_gcp_dns_flattened_8c4bcfef field: json target_field: gcp.dns.flattened if: ctx.json != null && ctx._conf?.keep_json == true - remove: + tag: remove_00070c09 field: - _conf - json ignore_missing: true - script: + tag: script_4d6993cb lang: painless description: This script processor iterates over the whole document to remove fields with null values. source: | @@ -378,3 +437,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/gcp/data_stream/firestore/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/firestore/elasticsearch/ingest_pipeline/default.yml index c79d82a6220..7ff04616596 100644 --- a/packages/gcp/data_stream/firestore/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/firestore/elasticsearch/ingest_pipeline/default.yml @@ -2,18 +2,22 @@ description: Pipeline for parsing GCP Firestore metrics. processors: - rename: + tag: rename_gcp_metrics_document_delete_count_to_gcp_firestore_document_delete_count_fabdfb5e field: gcp.metrics.document.delete.count target_field: gcp.firestore.document.delete.count ignore_missing: true - rename: + tag: rename_gcp_metrics_document_read_count_to_gcp_firestore_document_read_count_89516988 field: gcp.metrics.document.read.count target_field: gcp.firestore.document.read.count ignore_missing: true - rename: + tag: rename_gcp_metrics_document_write_count_to_gcp_firestore_document_write_count_2b376100 field: gcp.metrics.document.write.count target_field: gcp.firestore.document.write.count ignore_missing: true - remove: + tag: remove_ab6e04c7 field: - gcp.metrics ignore_missing: true @@ -24,3 +28,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/gcp/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml index 7de493b6709..9b828065139 100644 --- a/packages/gcp/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml @@ -3,143 +3,176 @@ description: Pipeline for Google Cloud Firewall Logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: 'ctx.event?.original == null' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - community_id: + tag: community_id_46d7956f source_ip: json.jsonPayload.connection.src_ip source_port: json.jsonPayload.connection.src_port destination_ip: json.jsonPayload.connection.dest_ip destination_port: json.jsonPayload.connection.dest_port iana_number: json.jsonPayload.connection.protocol - date: + tag: date_json_timestamp_e111e976 field: json.timestamp timezone: UTC formats: - ISO8601 - set: + tag: set_event_kind_de80643c field: event.kind value: event - append: + tag: append_event_category_7afdca3c field: event.category value: network - set: + tag: set_event_action_69a06be7 field: event.action value: firewall-rule - set: + tag: set_cloud_provider_62c5e42e field: cloud.provider value: gcp - set: + tag: set_log_logger_01230fab field: log.logger copy_from: json.logName ignore_failure: true - set: + tag: set_network_name_dd708766 field: network.name copy_from: json.resource.labels.subnetwork_name ignore_failure: true - set: + tag: set_event_id_efd37a25 field: event.id copy_from: json.insertId ignore_empty_value: true ignore_failure: true - lowercase: + tag: lowercase_json_jsonPayload_disposition_fb062e4c field: json.jsonPayload.disposition if: ctx?.json?.jsonPayload?.disposition != null - append: + tag: append_event_type_d5282502 field: event.type value: '{{json.jsonPayload.disposition}}' if: ctx?.json?.jsonPayload?.disposition != null - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - set: + tag: set_network_direction_6b8ae95b field: network.direction value: inbound if: ctx?.json?.jsonPayload?.rule_details?.direction == "INGRESS" - set: + tag: set_network_direction_2bb385cc field: network.direction value: outbound if: ctx?.json?.jsonPayload?.rule_details?.direction == "EGRESS" - set: + tag: set_network_direction_f672f5c9 field: network.direction value: unknown if: ctx?.network?.direction == null - set: + tag: set__jsonPayload_src_vpc_fea93931 field: _jsonPayload.src_vpc copy_from: json.jsonPayload.vpc if: ctx?.network?.direction == "outbound" ignore_failure: true - set: + tag: set__jsonPayload_src_instance_3c8adb4f field: _jsonPayload.src_instance copy_from: json.jsonPayload.instance if: ctx?.network?.direction == "outbound" ignore_failure: true - set: + tag: set__jsonPayload_src_location_d346c487 field: _jsonPayload.src_location copy_from: json.jsonPayload.location if: ctx?.network?.direction == "outbound" ignore_failure: true - set: + tag: set__jsonPayload_dest_vpc_d562441e field: _jsonPayload.dest_vpc copy_from: json.jsonPayload.remote_vpc if: ctx?.network?.direction == "outbound" ignore_failure: true - set: + tag: set__jsonPayload_dest_instance_7560ca28 field: _jsonPayload.dest_instance copy_from: json.jsonPayload.remote_instance if: ctx?.network?.direction == "outbound" ignore_failure: true - set: + tag: set__jsonPayload_dest_location_f6456040 field: _jsonPayload.dest_location copy_from: json.jsonPayload.remote_location if: ctx?.network?.direction == "outbound" ignore_failure: true - set: + tag: set__jsonPayload_dest_vpc_ae81acea field: _jsonPayload.dest_vpc copy_from: json.jsonPayload.vpc if: ctx?.network?.direction == "inbound" ignore_failure: true - set: + tag: set__jsonPayload_dest_instance_94854328 field: _jsonPayload.dest_instance copy_from: json.jsonPayload.instance if: ctx?.network?.direction == "inbound" ignore_failure: true - set: + tag: set__jsonPayload_dest_location_7a87b440 field: _jsonPayload.dest_location copy_from: json.jsonPayload.location if: ctx?.network?.direction == "inbound" ignore_failure: true - set: + tag: set__jsonPayload_src_vpc_2299c961 field: _jsonPayload.src_vpc copy_from: json.jsonPayload.remote_vpc if: ctx?.network?.direction == "inbound" ignore_failure: true - set: + tag: set__jsonPayload_src_instance_edebfabb field: _jsonPayload.src_instance copy_from: json.jsonPayload.remote_instance if: ctx?.network?.direction == "inbound" ignore_failure: true - set: + tag: set__jsonPayload_src_location_1724ba3b field: _jsonPayload.src_location copy_from: json.jsonPayload.remote_location if: ctx?.network?.direction == "inbound" ignore_failure: true - set: + tag: set_network_iana_number_1bb4e74b field: network.iana_number copy_from: json.jsonPayload.connection.protocol ignore_failure: true - convert: + tag: convert_network_iana_number_b5dcb911 field: network.iana_number type: string ignore_missing: true - script: + tag: script_aaef8fa9 lang: painless ignore_failure: true if: ctx?.network?.iana_number != null @@ -169,125 +202,153 @@ processors: ctx.network.transport = 'sctp'; } - set: + tag: set_destination_address_3f9e1c04 field: destination.address copy_from: json.jsonPayload.connection.dest_ip ignore_failure: true - set: + tag: set_destination_port_48df1199 field: destination.port copy_from: json.jsonPayload.connection.dest_port ignore_failure: true - set: + tag: set_source_address_a1ff7ced field: source.address copy_from: json.jsonPayload.connection.src_ip ignore_failure: true - set: + tag: set_source_port_0090c682 field: source.port copy_from: json.jsonPayload.connection.src_port ignore_failure: true - rename: + tag: rename__jsonPayload_src_instance_vm_name_to_source_domain_a1cf65f1 field: _jsonPayload.src_instance.vm_name target_field: source.domain ignore_missing: true - rename: + tag: rename__jsonPayload_dest_instance_vm_name_to_destination_domain_595698ea field: _jsonPayload.dest_instance.vm_name target_field: destination.domain ignore_missing: true - rename: + tag: rename__jsonPayload_dest_location_asn_to_destination_as_number_39012dd0 field: _jsonPayload.dest_location.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename__jsonPayload_dest_location_continent_to_destination_geo_continent_name_b0ee6f70 field: _jsonPayload.dest_location.continent target_field: destination.geo.continent_name ignore_missing: true - rename: + tag: rename__jsonPayload_dest_location_country_to_destination_geo_country_name_6827c0e4 field: _jsonPayload.dest_location.country target_field: destination.geo.country_name ignore_missing: true - rename: + tag: rename__jsonPayload_dest_location_region_to_destination_geo_region_name_ddff3830 field: _jsonPayload.dest_location.region target_field: destination.geo.region_name ignore_missing: true - rename: + tag: rename__jsonPayload_dest_location_city_to_destination_geo_city_name_4d8f9698 field: _jsonPayload.dest_location.city target_field: destination.geo.city_name ignore_missing: true - rename: + tag: rename__jsonPayload_src_location_asn_to_source_as_number_53b97437 field: _jsonPayload.src_location.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename__jsonPayload_src_location_continent_to_source_geo_continent_name_c432ef71 field: _jsonPayload.src_location.continent target_field: source.geo.continent_name ignore_missing: true - rename: + tag: rename__jsonPayload_src_location_country_to_source_geo_country_name_b3890879 field: _jsonPayload.src_location.country target_field: source.geo.country_name ignore_missing: true - rename: + tag: rename__jsonPayload_src_location_region_to_source_geo_region_name_3669d90f field: _jsonPayload.src_location.region target_field: source.geo.region_name ignore_missing: true - rename: + tag: rename__jsonPayload_src_location_city_to_source_geo_city_name_cd692cc1 field: _jsonPayload.src_location.city target_field: source.geo.city_name ignore_missing: true - rename: + tag: rename__jsonPayload_dest_instance_to_gcp_destination_instance_1b5fe576 field: _jsonPayload.dest_instance target_field: gcp.destination.instance ignore_missing: true - rename: + tag: rename__jsonPayload_dest_vpc_to_gcp_destination_vpc_ecae5db6 field: _jsonPayload.dest_vpc target_field: gcp.destination.vpc ignore_missing: true - rename: + tag: rename__jsonPayload_src_instance_to_gcp_source_instance_afb3e97b field: _jsonPayload.src_instance target_field: gcp.source.instance ignore_missing: true - rename: + tag: rename__jsonPayload_src_vpc_to_gcp_source_vpc_f032135d field: _jsonPayload.src_vpc target_field: gcp.source.vpc ignore_missing: true - rename: + tag: rename_json_jsonPayload_rule_details_reference_to_rule_name_bb616be5 field: json.jsonPayload.rule_details.reference target_field: rule.name ignore_missing: true - set: + tag: set_source_ip_1d3ed944 field: source.ip value: "{{source.address}}" if: ctx?.source?.address != null ignore_failure: true - set: + tag: set_destination_ip_2567920d field: destination.ip value: "{{destination.address}}" if: ctx?.destination?.address != null ignore_failure: true - convert: + tag: convert_gcp_source_instance_project_id_to_cloud_project_id_3cf3531b field: gcp.source.instance.project_id target_field: cloud.project.id type: string ignore_missing: true if: ctx?.network?.direction == "outbound" - convert: + tag: convert_gcp_source_instance_vm_name_to_cloud_instance_name_26bbd735 field: gcp.source.instance.vm_name target_field: cloud.instance.name type: string ignore_missing: true if: ctx?.network?.direction == "outbound" - convert: + tag: convert_gcp_source_instance_region_to_cloud_region_ececb5a4 field: gcp.source.instance.region target_field: cloud.region type: string ignore_missing: true if: ctx?.network?.direction == "outbound" - convert: + tag: convert_gcp_source_instance_zone_to_cloud_availability_zone_f2b9ef0e field: gcp.source.instance.zone target_field: cloud.availability_zone type: string ignore_missing: true if: ctx?.network?.direction == "outbound" - convert: + tag: convert_gcp_source_vpc_subnetwork_name_to_network_name_b36e992b field: gcp.source.vpc.subnetwork_name target_field: network.name type: string @@ -295,30 +356,35 @@ processors: ignore_failure: true if: ctx?.network?.direction == "outbound" - convert: + tag: convert_gcp_destination_instance_project_id_to_cloud_project_id_ca298ca7 field: gcp.destination.instance.project_id target_field: cloud.project.id type: string ignore_missing: true if: ctx?.network?.direction == "inbound" - convert: + tag: convert_gcp_destination_instance_vm_name_to_cloud_instance_name_3f774beb field: gcp.destination.instance.vm_name target_field: cloud.instance.name type: string ignore_missing: true if: ctx?.network?.direction == "inbound" - convert: + tag: convert_gcp_destination_instance_region_to_cloud_region_23ec5daa field: gcp.destination.instance.region target_field: cloud.region type: string ignore_missing: true if: ctx?.network?.direction == "inbound" - convert: + tag: convert_gcp_destination_instance_zone_to_cloud_availability_zone_625d2a14 field: gcp.destination.instance.zone target_field: cloud.availability_zone type: string ignore_missing: true if: ctx?.network?.direction == "inbound" - convert: + tag: convert_gcp_destination_vpc_subnetwork_name_to_network_name_9eef67bd field: gcp.destination.vpc.subnetwork_name target_field: network.name type: string @@ -326,36 +392,44 @@ processors: ignore_failure: true if: ctx?.network?.direction == "inbound" - set: + tag: set_network_direction_14948af4 field: network.direction value: internal if: ctx?.gcp?.source?.instance == ctx?.gcp?.destination?.instance - set: + tag: set_network_type_409f99d1 field: network.type value: ipv4 - if: ctx?.source?.ip != null && ctx?.source?.ip.contains(".") + if: ctx?.source?.ip != null && ctx?.source?.ip.contains(".") - set: + tag: set_network_type_3a24ca9a field: network.type value: ipv6 if: ctx?.source?.ip != null && !ctx?.source?.ip.contains(".") - set: + tag: set_gcp_firewall_rule_details_b37c34ee field: gcp.firewall.rule_details copy_from: json.jsonPayload.rule_details ignore_failure: true - append: + tag: append_related_ip_c40a2e39 field: related.ip value: "{{source.ip}}" allow_duplicates: false if: ctx?.source?.ip != null && ctx?.source?.ip != "" - append: + tag: append_related_ip_b730887a field: related.ip value: "{{destination.ip}}" allow_duplicates: false if: ctx?.destination?.ip != null && ctx?.destination?.ip != "" - rename: + tag: rename_json_to_gcp_firewall_flattened_5c64c5b8 field: json target_field: gcp.firewall.flattened if: ctx.json != null && ctx._conf?.keep_json == true - remove: + tag: remove_f2d436c9 field: - gcp.firewall.connection - gcp.firewall.dest_location @@ -367,15 +441,18 @@ processors: ignore_missing: true # IP Geolocation Lookup - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true # IP Autonomous System (AS) Lookup - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -384,6 +461,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -392,18 +470,22 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true @@ -414,3 +496,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/gcp/data_stream/gke/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/gke/elasticsearch/ingest_pipeline/default.yml index 6b389d11e4e..687b4ea1e85 100644 --- a/packages/gcp/data_stream/gke/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/gke/elasticsearch/ingest_pipeline/default.yml @@ -2,171 +2,213 @@ description: Pipeline for parsing GCP GKE metrics. processors: - fingerprint: + tag: fingerprint_5c1d1252 fields: - gcp.labels target_field: gcp.labels_fingerprint ignore_missing: true - rename: + tag: rename_gcp_metrics_container_cpu_core_usage_time_sec_to_gcp_gke_container_cpu_core_usage_time_sec_508ba6ce field: gcp.metrics.container.cpu.core_usage_time.sec target_field: gcp.gke.container.cpu.core_usage_time.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_container_cpu_limit_cores_value_to_gcp_gke_container_cpu_limit_cores_value_69e74900 field: gcp.metrics.container.cpu.limit_cores.value target_field: gcp.gke.container.cpu.limit_cores.value ignore_missing: true - rename: + tag: rename_gcp_metrics_container_cpu_limit_utilization_pct_to_gcp_gke_container_cpu_limit_utilization_pct_d980e5ac field: gcp.metrics.container.cpu.limit_utilization.pct target_field: gcp.gke.container.cpu.limit_utilization.pct ignore_missing: true - rename: + tag: rename_gcp_metrics_container_cpu_request_cores_value_to_gcp_gke_container_cpu_request_cores_value_b6a5be6c field: gcp.metrics.container.cpu.request_cores.value target_field: gcp.gke.container.cpu.request_cores.value ignore_missing: true - rename: + tag: rename_gcp_metrics_container_cpu_request_utilization_pct_to_gcp_gke_container_cpu_request_utilization_pct_a36a0c9c field: gcp.metrics.container.cpu.request_utilization.pct target_field: gcp.gke.container.cpu.request_utilization.pct ignore_missing: true - rename: + tag: rename_gcp_metrics_container_ephemeral_storage_limit_bytes_to_gcp_gke_container_ephemeral_storage_limit_bytes_42dade24 field: gcp.metrics.container.ephemeral_storage.limit.bytes target_field: gcp.gke.container.ephemeral_storage.limit.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_container_ephemeral_storage_request_bytes_to_gcp_gke_container_ephemeral_storage_request_bytes_b1afeed4 field: gcp.metrics.container.ephemeral_storage.request.bytes target_field: gcp.gke.container.ephemeral_storage.request.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_container_ephemeral_storage_used_bytes_to_gcp_gke_container_ephemeral_storage_used_bytes_c7b0b2bc field: gcp.metrics.container.ephemeral_storage.used.bytes target_field: gcp.gke.container.ephemeral_storage.used.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_container_memory_limit_bytes_to_gcp_gke_container_memory_limit_bytes_a63896d4 field: gcp.metrics.container.memory.limit.bytes target_field: gcp.gke.container.memory.limit.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_container_memory_limit_utilization_pct_to_gcp_gke_container_memory_limit_utilization_pct_1e9f8a74 field: gcp.metrics.container.memory.limit_utilization.pct target_field: gcp.gke.container.memory.limit_utilization.pct ignore_missing: true - rename: + tag: rename_gcp_metrics_container_memory_page_fault_count_to_gcp_gke_container_memory_page_fault_count_2ff08852 field: gcp.metrics.container.memory.page_fault.count target_field: gcp.gke.container.memory.page_fault.count ignore_missing: true - rename: + tag: rename_gcp_metrics_container_memory_request_bytes_to_gcp_gke_container_memory_request_bytes_c564f7e0 field: gcp.metrics.container.memory.request.bytes target_field: gcp.gke.container.memory.request.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_container_memory_request_utilization_pct_to_gcp_gke_container_memory_request_utilization_pct_7e398c04 field: gcp.metrics.container.memory.request_utilization.pct target_field: gcp.gke.container.memory.request_utilization.pct ignore_missing: true - rename: + tag: rename_gcp_metrics_container_memory_used_bytes_to_gcp_gke_container_memory_used_bytes_75213880 field: gcp.metrics.container.memory.used.bytes target_field: gcp.gke.container.memory.used.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_container_restart_count_to_gcp_gke_container_restart_count_1dbeebc2 field: gcp.metrics.container.restart.count target_field: gcp.gke.container.restart.count ignore_missing: true - rename: + tag: rename_gcp_metrics_container_uptime_sec_to_gcp_gke_container_uptime_sec_ce3de934 field: gcp.metrics.container.uptime.sec target_field: gcp.gke.container.uptime.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_node_cpu_allocatable_cores_value_to_gcp_gke_node_cpu_allocatable_cores_value_d1cf7c34 field: gcp.metrics.node.cpu.allocatable_cores.value target_field: gcp.gke.node.cpu.allocatable_cores.value ignore_missing: true - rename: + tag: rename_gcp_metrics_node_cpu_allocatable_utilization_pct_to_gcp_gke_node_cpu_allocatable_utilization_pct_ebce2e64 field: gcp.metrics.node.cpu.allocatable_utilization.pct target_field: gcp.gke.node.cpu.allocatable_utilization.pct ignore_missing: true - rename: + tag: rename_gcp_metrics_node_cpu_core_usage_time_sec_to_gcp_gke_node_cpu_core_usage_time_sec_2acf3d24 field: gcp.metrics.node.cpu.core_usage_time.sec target_field: gcp.gke.node.cpu.core_usage_time.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_node_cpu_total_cores_value_to_gcp_gke_node_cpu_total_cores_value_ceac83fc field: gcp.metrics.node.cpu.total_cores.value target_field: gcp.gke.node.cpu.total_cores.value ignore_missing: true - rename: + tag: rename_gcp_metrics_node_ephemeral_storage_allocatable_bytes_to_gcp_gke_node_ephemeral_storage_allocatable_bytes_5549e09c field: gcp.metrics.node.ephemeral_storage.allocatable.bytes target_field: gcp.gke.node.ephemeral_storage.allocatable.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_node_ephemeral_storage_inodes_free_value_to_gcp_gke_node_ephemeral_storage_inodes_free_value_33366b4c field: gcp.metrics.node.ephemeral_storage.inodes_free.value target_field: gcp.gke.node.ephemeral_storage.inodes_free.value ignore_missing: true - rename: + tag: rename_gcp_metrics_node_ephemeral_storage_inodes_total_value_to_gcp_gke_node_ephemeral_storage_inodes_total_value_d01b3ae2 field: gcp.metrics.node.ephemeral_storage.inodes_total.value target_field: gcp.gke.node.ephemeral_storage.inodes_total.value ignore_missing: true - rename: + tag: rename_gcp_metrics_node_ephemeral_storage_total_bytes_to_gcp_gke_node_ephemeral_storage_total_bytes_937a4ae0 field: gcp.metrics.node.ephemeral_storage.total.bytes target_field: gcp.gke.node.ephemeral_storage.total.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_node_ephemeral_storage_used_bytes_to_gcp_gke_node_ephemeral_storage_used_bytes_771b4d86 field: gcp.metrics.node.ephemeral_storage.used.bytes target_field: gcp.gke.node.ephemeral_storage.used.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_node_memory_allocatable_bytes_to_gcp_gke_node_memory_allocatable_bytes_1c4e039c field: gcp.metrics.node.memory.allocatable.bytes target_field: gcp.gke.node.memory.allocatable.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_node_memory_allocatable_utilization_pct_to_gcp_gke_node_memory_allocatable_utilization_pct_efc5c6b6 field: gcp.metrics.node.memory.allocatable_utilization.pct target_field: gcp.gke.node.memory.allocatable_utilization.pct ignore_missing: true - rename: + tag: rename_gcp_metrics_node_memory_total_bytes_to_gcp_gke_node_memory_total_bytes_53a9b084 field: gcp.metrics.node.memory.total.bytes target_field: gcp.gke.node.memory.total.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_node_memory_used_bytes_to_gcp_gke_node_memory_used_bytes_9d6ab4d4 field: gcp.metrics.node.memory.used.bytes target_field: gcp.gke.node.memory.used.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_node_network_received_bytes_to_gcp_gke_node_network_received_bytes_43dd2484 field: gcp.metrics.node.network.received.bytes target_field: gcp.gke.node.network.received.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_node_network_sent_bytes_to_gcp_gke_node_network_sent_bytes_7084638a field: gcp.metrics.node.network.sent.bytes target_field: gcp.gke.node.network.sent.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_node_pid_limit_value_to_gcp_gke_node_pid_limit_value_e7cd9e6c field: gcp.metrics.node.pid_limit.value target_field: gcp.gke.node.pid_limit.value ignore_missing: true - rename: + tag: rename_gcp_metrics_node_pid_used_value_to_gcp_gke_node_pid_used_value_a127b410 field: gcp.metrics.node.pid_used.value target_field: gcp.gke.node.pid_used.value ignore_missing: true - rename: + tag: rename_gcp_metrics_node_daemon_cpu_core_usage_time_sec_to_gcp_gke_node_daemon_cpu_core_usage_time_sec_bdfb38c6 field: gcp.metrics.node_daemon.cpu.core_usage_time.sec target_field: gcp.gke.node_daemon.cpu.core_usage_time.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_node_daemon_memory_used_bytes_to_gcp_gke_node_daemon_memory_used_bytes_505f4c64 field: gcp.metrics.node_daemon.memory.used.bytes target_field: gcp.gke.node_daemon.memory.used.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_pod_network_received_bytes_to_gcp_gke_pod_network_received_bytes_8f540234 field: gcp.metrics.pod.network.received.bytes target_field: gcp.gke.pod.network.received.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_pod_network_sent_bytes_to_gcp_gke_pod_network_sent_bytes_e7ea7944 field: gcp.metrics.pod.network.sent.bytes target_field: gcp.gke.pod.network.sent.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_pod_volume_total_bytes_to_gcp_gke_pod_volume_total_bytes_73abd574 field: gcp.metrics.pod.volume.total.bytes target_field: gcp.gke.pod.volume.total.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_pod_volume_used_bytes_to_gcp_gke_pod_volume_used_bytes_a3056bca field: gcp.metrics.pod.volume.used.bytes target_field: gcp.gke.pod.volume.used.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_pod_volume_utilization_pct_to_gcp_gke_pod_volume_utilization_pct_50375f90 field: gcp.metrics.pod.volume.utilization.pct target_field: gcp.gke.pod.volume.utilization.pct ignore_missing: true - remove: + tag: remove_ab6e04c7 field: - gcp.metrics ignore_missing: true @@ -177,3 +219,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/gcp/data_stream/loadbalancing_logs/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/loadbalancing_logs/elasticsearch/ingest_pipeline/default.yml index 4baf0fd766c..4644fcdc7db 100644 --- a/packages/gcp/data_stream/loadbalancing_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/loadbalancing_logs/elasticsearch/ingest_pipeline/default.yml @@ -3,71 +3,87 @@ description: Pipeline for Google Cloud DNS logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: 'ctx.event?.original == null' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_event_kind_de80643c field: event.kind value: event - append: + tag: append_event_category_2c5cae23 field: event.category value: ["network"] allow_duplicates: false - append: + tag: append_event_type_6adea021 field: event.type value: ["info"] allow_duplicates: false - date: + tag: date_json_timestamp_e111e976 field: json.timestamp timezone: UTC formats: - ISO8601 - date: + tag: date_json_receiveTimestamp_to_event_created_fd746809 field: json.receiveTimestamp target_field: event.created timezone: UTC formats: - ISO8601 - rename: + tag: rename_json_logName_to_log_logger_73f0a6eb field: json.logName target_field: log.logger ignore_missing: true - - set: + - set: + tag: set_event_id_efd37a25 field: event.id copy_from: json.insertId ignore_empty_value: true ignore_failure: true - - convert: + - convert: + tag: convert_json_resource_labels_project_id_to_cloud_project_id_a91d4841 field: json.resource.labels.project_id target_field: cloud.project.id type: string ignore_failure: true - - convert: + - convert: + tag: convert_json_resource_labels_zone_to_cloud_region_2857d2fa field: json.resource.labels.zone target_field: cloud.region type: string ignore_failure: true - grok: + tag: grok_json_httpRequest_remoteIp_57caf45b field: json.httpRequest.remoteIp ignore_missing: true patterns: - ^%{IP:source.address}(:%{POSINT:source.port:long})?$ - convert: + tag: convert_source_address_to_source_ip_0a89ba51 field: source.address target_field: source.ip type: ip ignore_failure: true - geoip: + tag: geoip_source_ip_to_source_geo_0494c6b1 field: source.ip target_field: source.geo - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -76,135 +92,165 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_json_httpRequest_requestMethod_to_http_request_method_04af1619 field: json.httpRequest.requestMethod target_field: http.request.method ignore_missing: true - convert: + tag: convert_json_httpRequest_requestSize_to_http_request_bytes_edcd4360 field: json.httpRequest.requestSize target_field: http.request.bytes type: long ignore_missing: true - convert: + tag: convert_json_httpRequest_responseSize_to_http_response_bytes_6ab6bb76 field: json.httpRequest.responseSize target_field: http.response.bytes type: long ignore_missing: true - rename: + tag: rename_json_httpRequest_status_to_http_response_status_code_5fa5c66a field: json.httpRequest.status target_field: http.response.status_code ignore_missing: true - dissect: + tag: dissect_json_httpRequest_protocol_a1a3bd33 field: json.httpRequest.protocol pattern: "%{network.protocol}/%{http.version}" ignore_failure: true if: ctx.json?.httpRequest?.protocol != null - lowercase: + tag: lowercase_network_protocol_49872259 field: network.protocol ignore_missing: true - user_agent: + tag: user_agent_json_httpRequest_userAgent_to_user_agent_bc3f8b77 field: json.httpRequest.userAgent target_field: user_agent ignore_missing: true - uri_parts: + tag: uri_parts_json_httpRequest_requestUrl_to_url_6132c0a8 field: json.httpRequest.requestUrl target_field: url if: ctx.json?.httpRequest?.requestUrl != null - rename: + tag: rename_json_httpRequest_referer_to_http_request_referrer_67907f66 field: json.httpRequest.referer target_field: http.request.referrer ignore_missing: true - grok: + tag: grok_json_httpRequest_serverIp_be0d27c1 field: json.httpRequest.serverIp ignore_missing: true patterns: - ^%{IP:destination.nat.ip}(:%{POSINT:destination.nat.port:long})?$ - - set: + - set: + tag: set_destination_address_e3fe5355 field: destination.address copy_from: url.domain ignore_empty_value: true ignore_failure: true - - set: + - set: + tag: set_destination_port_e5981b43 field: destination.port copy_from: url.port ignore_empty_value: true ignore_failure: true - convert: + tag: convert_destination_address_to_destination_ip_602d8632 field: destination.address target_field: destination.ip type: ip ignore_missing: true on_failure: - - rename: + - rename: + tag: rename_url_domain_to_destination_domain_33ca3ed3 field: url.domain target_field: destination.domain - ignore_missing: true + ignore_missing: true ignore_failure: true - rename: + tag: rename_json_severity_to_log_level_db5923df field: json.severity target_field: log.level ignore_missing: true - rename: + tag: rename_json_jsonPayload_cacheId_to_gcp_load_balancer_cache_id_9a294e58 field: json.jsonPayload.cacheId target_field: gcp.load_balancer.cache_id ignore_missing: true - rename: + tag: rename_json_jsonPayload_statusDetails_to_gcp_load_balancer_status_details_84210076 field: json.jsonPayload.statusDetails target_field: gcp.load_balancer.status_details ignore_missing: true - rename: + tag: rename_json_httpRequest_cacheHit_to_gcp_load_balancer_cache_hit_9f52461f field: json.httpRequest.cacheHit target_field: gcp.load_balancer.cache_hit ignore_missing: true - rename: + tag: rename_json_httpRequest_cacheLookup_to_gcp_load_balancer_cache_lookup_a949b333 field: json.httpRequest.cacheLookup target_field: gcp.load_balancer.cache_lookup ignore_missing: true - rename: + tag: rename_json_resource_labels_url_map_name_to_gcp_load_balancer_url_map_name_0aca1d08 field: json.resource.labels.url_map_name target_field: gcp.load_balancer.url_map_name ignore_missing: true - rename: + tag: rename_json_resource_labels_forwarding_rule_name_to_gcp_load_balancer_forwarding_rule_name_0a2ca280 field: json.resource.labels.forwarding_rule_name target_field: gcp.load_balancer.forwarding_rule_name ignore_missing: true - rename: + tag: rename_json_resource_labels_target_proxy_name_to_gcp_load_balancer_target_proxy_name_c7835324 field: json.resource.labels.target_proxy_name target_field: gcp.load_balancer.target_proxy_name ignore_missing: true - rename: + tag: rename_json_resource_labels_backend_service_name_to_gcp_load_balancer_backend_service_name_75d06e38 field: json.resource.labels.backend_service_name target_field: gcp.load_balancer.backend_service_name ignore_missing: true - append: + tag: append_related_ip_a2836bfa field: related.ip value: "{{source.ip}}" allow_duplicates: false if: ctx?.source?.ip != null - append: + tag: append_related_ip_bd4aa684 field: related.ip value: "{{destination.ip}}" allow_duplicates: false if: ctx?.destination?.ip != null - append: + tag: append_related_ip_2c2fa2bf field: related.ip value: "{{destination.nat.ip}}" allow_duplicates: false if: ctx?.destination?.nat?.ip != null - append: + tag: append_related_hosts_c4bb4f1e field: related.hosts value: "{{destination.domain}}" allow_duplicates: false if: ctx?.destination?.domain != null - remove: - field: + tag: remove_e5a80934 + field: - json ignore_missing: true on_failure: @@ -214,3 +260,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/gcp/data_stream/loadbalancing_metrics/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/loadbalancing_metrics/elasticsearch/ingest_pipeline/default.yml index 982e1b65ee4..22bbeec8b2b 100644 --- a/packages/gcp/data_stream/loadbalancing_metrics/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/loadbalancing_metrics/elasticsearch/ingest_pipeline/default.yml @@ -2,127 +2,158 @@ description: Pipeline for parsing GCP Loadbalancing metrics. processors: - fingerprint: + tag: fingerprint_5c1d1252 fields: - gcp.labels target_field: gcp.labels_fingerprint ignore_missing: true - rename: + tag: rename_gcp_metrics_https_backend_request_bytes_to_gcp_loadbalancing_metrics_https_backend_request_bytes_a6bfb0a4 field: gcp.metrics.https.backend_request.bytes target_field: gcp.loadbalancing_metrics.https.backend_request.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_https_backend_request_count_to_gcp_loadbalancing_metrics_https_backend_request_count_fac9a9e4 field: gcp.metrics.https.backend_request.count target_field: gcp.loadbalancing_metrics.https.backend_request.count ignore_missing: true - rename: + tag: rename_gcp_metrics_https_backend_response_bytes_to_gcp_loadbalancing_metrics_https_backend_response_bytes_5ca2cf84 field: gcp.metrics.https.backend_response.bytes target_field: gcp.loadbalancing_metrics.https.backend_response.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_https_request_bytes_to_gcp_loadbalancing_metrics_https_request_bytes_a0a5e3b2 field: gcp.metrics.https.request.bytes target_field: gcp.loadbalancing_metrics.https.request.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_https_request_count_to_gcp_loadbalancing_metrics_https_request_count_3033cf3a field: gcp.metrics.https.request.count target_field: gcp.loadbalancing_metrics.https.request.count ignore_missing: true - rename: + tag: rename_gcp_metrics_https_response_bytes_to_gcp_loadbalancing_metrics_https_response_bytes_4d37daf0 field: gcp.metrics.https.response.bytes target_field: gcp.loadbalancing_metrics.https.response.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_l3_external_egress_bytes_to_gcp_loadbalancing_metrics_l3_external_egress_bytes_1427be40 field: gcp.metrics.l3.external.egress.bytes target_field: gcp.loadbalancing_metrics.l3.external.egress.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_l3_external_egress_packets_count_to_gcp_loadbalancing_metrics_l3_external_egress_packets_count_6843f684 field: gcp.metrics.l3.external.egress_packets.count target_field: gcp.loadbalancing_metrics.l3.external.egress_packets.count ignore_missing: true - rename: + tag: rename_gcp_metrics_l3_external_ingress_bytes_to_gcp_loadbalancing_metrics_l3_external_ingress_bytes_69a292e8 field: gcp.metrics.l3.external.ingress.bytes target_field: gcp.loadbalancing_metrics.l3.external.ingress.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_l3_external_ingress_packets_count_to_gcp_loadbalancing_metrics_l3_external_ingress_packets_count_452f12bc field: gcp.metrics.l3.external.ingress_packets.count target_field: gcp.loadbalancing_metrics.l3.external.ingress_packets.count ignore_missing: true - rename: + tag: rename_gcp_metrics_l3_internal_egress_bytes_to_gcp_loadbalancing_metrics_l3_internal_egress_bytes_e2ed2ce4 field: gcp.metrics.l3.internal.egress.bytes target_field: gcp.loadbalancing_metrics.l3.internal.egress.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_l3_internal_egress_packets_count_to_gcp_loadbalancing_metrics_l3_internal_egress_packets_count_9ce33958 field: gcp.metrics.l3.internal.egress_packets.count target_field: gcp.loadbalancing_metrics.l3.internal.egress_packets.count ignore_missing: true - rename: + tag: rename_gcp_metrics_l3_internal_ingress_bytes_to_gcp_loadbalancing_metrics_l3_internal_ingress_bytes_fccbef44 field: gcp.metrics.l3.internal.ingress.bytes target_field: gcp.loadbalancing_metrics.l3.internal.ingress.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_l3_internal_ingress_packets_count_to_gcp_loadbalancing_metrics_l3_internal_ingress_packets_count_6df75b68 field: gcp.metrics.l3.internal.ingress_packets.count target_field: gcp.loadbalancing_metrics.l3.internal.ingress_packets.count ignore_missing: true - rename: + tag: rename_gcp_metrics_tcp_ssl_proxy_closed_connections_value_to_gcp_loadbalancing_metrics_tcp_ssl_proxy_closed_connections_value_09b04474 field: gcp.metrics.tcp_ssl_proxy.closed_connections.value target_field: gcp.loadbalancing_metrics.tcp_ssl_proxy.closed_connections.value ignore_missing: true - rename: + tag: rename_gcp_metrics_tcp_ssl_proxy_egress_bytes_to_gcp_loadbalancing_metrics_tcp_ssl_proxy_egress_bytes_e4f53270 field: gcp.metrics.tcp_ssl_proxy.egress.bytes target_field: gcp.loadbalancing_metrics.tcp_ssl_proxy.egress.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_tcp_ssl_proxy_ingress_bytes_to_gcp_loadbalancing_metrics_tcp_ssl_proxy_ingress_bytes_27cfbaaa field: gcp.metrics.tcp_ssl_proxy.ingress.bytes target_field: gcp.loadbalancing_metrics.tcp_ssl_proxy.ingress.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_tcp_ssl_proxy_new_connections_value_to_gcp_loadbalancing_metrics_tcp_ssl_proxy_new_connections_value_92fbdb14 field: gcp.metrics.tcp_ssl_proxy.new_connections.value target_field: gcp.loadbalancing_metrics.tcp_ssl_proxy.new_connections.value ignore_missing: true - rename: + tag: rename_gcp_metrics_tcp_ssl_proxy_open_connections_value_to_gcp_loadbalancing_metrics_tcp_ssl_proxy_open_connections_value_0df0062c field: gcp.metrics.tcp_ssl_proxy.open_connections.value target_field: gcp.loadbalancing_metrics.tcp_ssl_proxy.open_connections.value ignore_missing: true - rename: + tag: rename_gcp_metrics_https_backend_latencies_value_to_gcp_loadbalancing_metrics_https_backend_latencies_value_89486802 field: gcp.metrics.https.backend_latencies.value target_field: gcp.loadbalancing_metrics.https.backend_latencies.value ignore_missing: true - rename: + tag: rename_gcp_metrics_https_external_regional_backend_latencies_value_to_gcp_loadbalancing_metrics_https_external_regional_backend_latencies_value_1d3872d6 field: gcp.metrics.https.external.regional.backend_latencies.value target_field: gcp.loadbalancing_metrics.https.external.regional.backend_latencies.value ignore_missing: true - rename: + tag: rename_gcp_metrics_https_external_regional_total_latencies_value_to_gcp_loadbalancing_metrics_https_external_regional_total_latencies_value_9a746d76 field: gcp.metrics.https.external.regional.total_latencies.value target_field: gcp.loadbalancing_metrics.https.external.regional.total_latencies.value ignore_missing: true - rename: + tag: rename_gcp_metrics_https_frontend_tcp_rtt_value_to_gcp_loadbalancing_metrics_https_frontend_tcp_rtt_value_138d4634 field: gcp.metrics.https.frontend_tcp_rtt.value target_field: gcp.loadbalancing_metrics.https.frontend_tcp_rtt.value ignore_missing: true - rename: + tag: rename_gcp_metrics_https_internal_backend_latencies_value_to_gcp_loadbalancing_metrics_https_internal_backend_latencies_value_aa22ff44 field: gcp.metrics.https.internal.backend_latencies.value target_field: gcp.loadbalancing_metrics.https.internal.backend_latencies.value ignore_missing: true - rename: + tag: rename_gcp_metrics_https_internal_total_latencies_value_to_gcp_loadbalancing_metrics_https_internal_total_latencies_value_015d906c field: gcp.metrics.https.internal.total_latencies.value target_field: gcp.loadbalancing_metrics.https.internal.total_latencies.value ignore_missing: true - rename: + tag: rename_gcp_metrics_https_total_latencies_value_to_gcp_loadbalancing_metrics_https_total_latencies_value_d6c1b8e2 field: gcp.metrics.https.total_latencies.value target_field: gcp.loadbalancing_metrics.https.total_latencies.value ignore_missing: true - rename: + tag: rename_gcp_metrics_l3_external_rtt_latencies_value_to_gcp_loadbalancing_metrics_l3_external_rtt_latencies_value_7d44194c field: gcp.metrics.l3.external.rtt_latencies.value target_field: gcp.loadbalancing_metrics.l3.external.rtt_latencies.value ignore_missing: true - rename: + tag: rename_gcp_metrics_l3_internal_rtt_latencies_value_to_gcp_loadbalancing_metrics_l3_internal_rtt_latencies_value_799173f0 field: gcp.metrics.l3.internal.rtt_latencies.value target_field: gcp.loadbalancing_metrics.l3.internal.rtt_latencies.value ignore_missing: true - rename: + tag: rename_gcp_metrics_tcp_ssl_proxy_frontend_tcp_rtt_value_to_gcp_loadbalancing_metrics_tcp_ssl_proxy_frontend_tcp_rtt_value_b06e5188 field: gcp.metrics.tcp_ssl_proxy.frontend_tcp_rtt.value target_field: gcp.loadbalancing_metrics.tcp_ssl_proxy.frontend_tcp_rtt.value ignore_missing: true - remove: + tag: remove_ab6e04c7 field: - gcp.metrics ignore_missing: true @@ -133,3 +164,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/gcp/data_stream/pubsub/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/pubsub/elasticsearch/ingest_pipeline/default.yml index bd573857da0..e3fdb020538 100644 --- a/packages/gcp/data_stream/pubsub/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/pubsub/elasticsearch/ingest_pipeline/default.yml @@ -2,207 +2,258 @@ description: Pipeline for parsing GCP PubSub metrics. processors: - fingerprint: + tag: fingerprint_5c1d1252 fields: - gcp.labels target_field: gcp.labels_fingerprint ignore_missing: true - rename: + tag: rename_gcp_metrics_snapshot_backlog_bytes_to_gcp_pubsub_snapshot_backlog_bytes_33c8f8dc field: gcp.metrics.snapshot.backlog.bytes target_field: gcp.pubsub.snapshot.backlog.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_snapshot_backlog_bytes_by_region_bytes_to_gcp_pubsub_snapshot_backlog_bytes_by_region_bytes_1b0b7336 field: gcp.metrics.snapshot.backlog_bytes_by_region.bytes target_field: gcp.pubsub.snapshot.backlog_bytes_by_region.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_snapshot_config_updates_count_to_gcp_pubsub_snapshot_config_updates_count_0890b5f4 field: gcp.metrics.snapshot.config_updates.count target_field: gcp.pubsub.snapshot.config_updates.count ignore_missing: true - rename: + tag: rename_gcp_metrics_snapshot_num_messages_value_to_gcp_pubsub_snapshot_num_messages_value_88cffd7c field: gcp.metrics.snapshot.num_messages.value target_field: gcp.pubsub.snapshot.num_messages.value ignore_missing: true - rename: + tag: rename_gcp_metrics_snapshot_num_messages_by_region_value_to_gcp_pubsub_snapshot_num_messages_by_region_value_d114ab98 field: gcp.metrics.snapshot.num_messages_by_region.value target_field: gcp.pubsub.snapshot.num_messages_by_region.value ignore_missing: true - rename: + tag: rename_gcp_metrics_snapshot_oldest_message_age_sec_to_gcp_pubsub_snapshot_oldest_message_age_sec_f4d08bdc field: gcp.metrics.snapshot.oldest_message_age.sec target_field: gcp.pubsub.snapshot.oldest_message_age.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_snapshot_oldest_message_age_by_region_sec_to_gcp_pubsub_snapshot_oldest_message_age_by_region_sec_38d8da1c field: gcp.metrics.snapshot.oldest_message_age_by_region.sec target_field: gcp.pubsub.snapshot.oldest_message_age_by_region.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_ack_message_count_to_gcp_pubsub_subscription_ack_message_count_d684afea field: gcp.metrics.subscription.ack_message.count target_field: gcp.pubsub.subscription.ack_message.count ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_backlog_bytes_to_gcp_pubsub_subscription_backlog_bytes_719d08ba field: gcp.metrics.subscription.backlog.bytes target_field: gcp.pubsub.subscription.backlog.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_byte_cost_bytes_to_gcp_pubsub_subscription_byte_cost_bytes_4771ca74 field: gcp.metrics.subscription.byte_cost.bytes target_field: gcp.pubsub.subscription.byte_cost.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_config_updates_count_to_gcp_pubsub_subscription_config_updates_count_f573b6cc field: gcp.metrics.subscription.config_updates.count target_field: gcp.pubsub.subscription.config_updates.count ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_dead_letter_message_count_to_gcp_pubsub_subscription_dead_letter_message_count_7f96993e field: gcp.metrics.subscription.dead_letter_message.count target_field: gcp.pubsub.subscription.dead_letter_message.count ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_mod_ack_deadline_message_count_to_gcp_pubsub_subscription_mod_ack_deadline_message_count_ff3a4b20 field: gcp.metrics.subscription.mod_ack_deadline_message.count target_field: gcp.pubsub.subscription.mod_ack_deadline_message.count ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_mod_ack_deadline_message_operation_count_to_gcp_pubsub_subscription_mod_ack_deadline_message_operation_count_de135a18 field: gcp.metrics.subscription.mod_ack_deadline_message_operation.count target_field: gcp.pubsub.subscription.mod_ack_deadline_message_operation.count ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_mod_ack_deadline_request_count_to_gcp_pubsub_subscription_mod_ack_deadline_request_count_a744b268 field: gcp.metrics.subscription.mod_ack_deadline_request.count target_field: gcp.pubsub.subscription.mod_ack_deadline_request.count ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_num_outstanding_messages_value_to_gcp_pubsub_subscription_num_outstanding_messages_value_d30c43f4 field: gcp.metrics.subscription.num_outstanding_messages.value target_field: gcp.pubsub.subscription.num_outstanding_messages.value ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_num_undelivered_messages_value_to_gcp_pubsub_subscription_num_undelivered_messages_value_5466ef70 field: gcp.metrics.subscription.num_undelivered_messages.value target_field: gcp.pubsub.subscription.num_undelivered_messages.value ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_oldest_retained_acked_message_age_sec_to_gcp_pubsub_subscription_oldest_retained_acked_message_age_sec_35cf7b52 field: gcp.metrics.subscription.oldest_retained_acked_message_age.sec target_field: gcp.pubsub.subscription.oldest_retained_acked_message_age.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_oldest_retained_acked_message_age_by_region_value_to_gcp_pubsub_subscription_oldest_retained_acked_message_age_by_region_value_20e2c834 field: gcp.metrics.subscription.oldest_retained_acked_message_age_by_region.value target_field: gcp.pubsub.subscription.oldest_retained_acked_message_age_by_region.value ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_oldest_unacked_message_age_sec_to_gcp_pubsub_subscription_oldest_unacked_message_age_sec_d82694a0 field: gcp.metrics.subscription.oldest_unacked_message_age.sec target_field: gcp.pubsub.subscription.oldest_unacked_message_age.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_oldest_unacked_message_age_by_region_value_to_gcp_pubsub_subscription_oldest_unacked_message_age_by_region_value_f1029308 field: gcp.metrics.subscription.oldest_unacked_message_age_by_region.value target_field: gcp.pubsub.subscription.oldest_unacked_message_age_by_region.value ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_pull_ack_message_operation_count_to_gcp_pubsub_subscription_pull_ack_message_operation_count_4d15e94c field: gcp.metrics.subscription.pull_ack_message_operation.count target_field: gcp.pubsub.subscription.pull_ack_message_operation.count ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_pull_ack_request_count_to_gcp_pubsub_subscription_pull_ack_request_count_8cdde46c field: gcp.metrics.subscription.pull_ack_request.count target_field: gcp.pubsub.subscription.pull_ack_request.count ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_pull_message_operation_count_to_gcp_pubsub_subscription_pull_message_operation_count_ecbf71e0 field: gcp.metrics.subscription.pull_message_operation.count target_field: gcp.pubsub.subscription.pull_message_operation.count ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_pull_request_count_to_gcp_pubsub_subscription_pull_request_count_721d0c10 field: gcp.metrics.subscription.pull_request.count target_field: gcp.pubsub.subscription.pull_request.count ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_push_request_count_to_gcp_pubsub_subscription_push_request_count_27570fac field: gcp.metrics.subscription.push_request.count target_field: gcp.pubsub.subscription.push_request.count ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_retained_acked_bytes_to_gcp_pubsub_subscription_retained_acked_bytes_b7315c90 field: gcp.metrics.subscription.retained_acked.bytes target_field: gcp.pubsub.subscription.retained_acked.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_retained_acked_bytes_by_region_bytes_to_gcp_pubsub_subscription_retained_acked_bytes_by_region_bytes_95db0014 field: gcp.metrics.subscription.retained_acked_bytes_by_region.bytes target_field: gcp.pubsub.subscription.retained_acked_bytes_by_region.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_seek_request_count_to_gcp_pubsub_subscription_seek_request_count_4992ba08 field: gcp.metrics.subscription.seek_request.count target_field: gcp.pubsub.subscription.seek_request.count ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_sent_message_count_to_gcp_pubsub_subscription_sent_message_count_8852cf64 field: gcp.metrics.subscription.sent_message.count target_field: gcp.pubsub.subscription.sent_message.count ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_streaming_pull_ack_message_operation_count_to_gcp_pubsub_subscription_streaming_pull_ack_message_operation_count_7a86a350 field: gcp.metrics.subscription.streaming_pull_ack_message_operation.count target_field: gcp.pubsub.subscription.streaming_pull_ack_message_operation.count ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_streaming_pull_ack_request_count_to_gcp_pubsub_subscription_streaming_pull_ack_request_count_0c7daef8 field: gcp.metrics.subscription.streaming_pull_ack_request.count target_field: gcp.pubsub.subscription.streaming_pull_ack_request.count ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_streaming_pull_message_operation_count_to_gcp_pubsub_subscription_streaming_pull_message_operation_count_8c290604 field: gcp.metrics.subscription.streaming_pull_message_operation.count target_field: gcp.pubsub.subscription.streaming_pull_message_operation.count ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_streaming_pull_mod_ack_deadline_message_operation_count_to_gcp_pubsub_subscription_streaming_pull_mod_ack_deadline_message_operation_count_4678a3f4 field: gcp.metrics.subscription.streaming_pull_mod_ack_deadline_message_operation.count target_field: gcp.pubsub.subscription.streaming_pull_mod_ack_deadline_message_operation.count ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_streaming_pull_mod_ack_deadline_request_count_to_gcp_pubsub_subscription_streaming_pull_mod_ack_deadline_request_count_09e2de14 field: gcp.metrics.subscription.streaming_pull_mod_ack_deadline_request.count target_field: gcp.pubsub.subscription.streaming_pull_mod_ack_deadline_request.count ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_streaming_pull_response_count_to_gcp_pubsub_subscription_streaming_pull_response_count_90934e74 field: gcp.metrics.subscription.streaming_pull_response.count target_field: gcp.pubsub.subscription.streaming_pull_response.count ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_unacked_bytes_by_region_bytes_to_gcp_pubsub_subscription_unacked_bytes_by_region_bytes_adc53c30 field: gcp.metrics.subscription.unacked_bytes_by_region.bytes target_field: gcp.pubsub.subscription.unacked_bytes_by_region.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_topic_byte_cost_bytes_to_gcp_pubsub_topic_byte_cost_bytes_62b96d20 field: gcp.metrics.topic.byte_cost.bytes target_field: gcp.pubsub.topic.byte_cost.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_topic_config_updates_count_to_gcp_pubsub_topic_config_updates_count_053b35fe field: gcp.metrics.topic.config_updates.count target_field: gcp.pubsub.topic.config_updates.count ignore_missing: true - rename: + tag: rename_gcp_metrics_topic_message_sizes_bytes_to_gcp_pubsub_topic_message_sizes_bytes_a285e4d8 field: gcp.metrics.topic.message_sizes.bytes target_field: gcp.pubsub.topic.message_sizes.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_topic_oldest_retained_acked_message_age_by_region_value_to_gcp_pubsub_topic_oldest_retained_acked_message_age_by_region_value_0df2e754 field: gcp.metrics.topic.oldest_retained_acked_message_age_by_region.value target_field: gcp.pubsub.topic.oldest_retained_acked_message_age_by_region.value ignore_missing: true - rename: + tag: rename_gcp_metrics_topic_oldest_unacked_message_age_by_region_value_to_gcp_pubsub_topic_oldest_unacked_message_age_by_region_value_6fd7e0f4 field: gcp.metrics.topic.oldest_unacked_message_age_by_region.value target_field: gcp.pubsub.topic.oldest_unacked_message_age_by_region.value ignore_missing: true - rename: + tag: rename_gcp_metrics_topic_retained_acked_bytes_by_region_bytes_to_gcp_pubsub_topic_retained_acked_bytes_by_region_bytes_bd104444 field: gcp.metrics.topic.retained_acked_bytes_by_region.bytes target_field: gcp.pubsub.topic.retained_acked_bytes_by_region.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_topic_send_message_operation_count_to_gcp_pubsub_topic_send_message_operation_count_5a30e320 field: gcp.metrics.topic.send_message_operation.count target_field: gcp.pubsub.topic.send_message_operation.count ignore_missing: true - rename: + tag: rename_gcp_metrics_topic_send_request_count_to_gcp_pubsub_topic_send_request_count_3746a888 field: gcp.metrics.topic.send_request.count target_field: gcp.pubsub.topic.send_request.count ignore_missing: true - rename: + tag: rename_gcp_metrics_topic_streaming_pull_response_count_to_gcp_pubsub_topic_streaming_pull_response_count_cc035cd4 field: gcp.metrics.topic.streaming_pull_response.count target_field: gcp.pubsub.topic.streaming_pull_response.count ignore_missing: true - rename: + tag: rename_gcp_metrics_topic_unacked_bytes_by_region_bytes_to_gcp_pubsub_topic_unacked_bytes_by_region_bytes_85ceca3c field: gcp.metrics.topic.unacked_bytes_by_region.bytes target_field: gcp.pubsub.topic.unacked_bytes_by_region.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_ack_latencies_value_to_gcp_pubsub_subscription_ack_latencies_value_ab1fdd24 field: gcp.metrics.subscription.ack_latencies.value target_field: gcp.pubsub.subscription.ack_latencies.value ignore_missing: true - rename: + tag: rename_gcp_metrics_subscription_push_request_latencies_value_to_gcp_pubsub_subscription_push_request_latencies_value_691c1cac field: gcp.metrics.subscription.push_request_latencies.value target_field: gcp.pubsub.subscription.push_request_latencies.value ignore_missing: true - remove: + tag: remove_ab6e04c7 field: - gcp.metrics ignore_missing: true @@ -213,3 +264,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/gcp/data_stream/redis/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/redis/elasticsearch/ingest_pipeline/default.yml index 736f87b0450..495e727dec8 100644 --- a/packages/gcp/data_stream/redis/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/redis/elasticsearch/ingest_pipeline/default.yml @@ -2,131 +2,163 @@ description: Pipeline for parsing GCP Redis metrics. processors: - fingerprint: + tag: fingerprint_5c1d1252 fields: - gcp.labels target_field: gcp.labels_fingerprint ignore_missing: true - rename: + tag: rename_gcp_metrics_clients_blocked_value_to_gcp_redis_clients_blocked_count_55d04a0e field: gcp.metrics.clients.blocked.value target_field: gcp.redis.clients.blocked.count ignore_missing: true - rename: + tag: rename_gcp_metrics_clients_connected_value_to_gcp_redis_clients_connected_count_7926d428 field: gcp.metrics.clients.connected.value target_field: gcp.redis.clients.connected.count ignore_missing: true - rename: + tag: rename_gcp_metrics_commands_calls_value_to_gcp_redis_commands_calls_count_e3d84d98 field: gcp.metrics.commands.calls.value target_field: gcp.redis.commands.calls.count ignore_missing: true - rename: + tag: rename_gcp_metrics_commands_total_time_value_to_gcp_redis_commands_total_time_us_de1b6479 field: gcp.metrics.commands.total_time.value target_field: gcp.redis.commands.total_time.us ignore_missing: true - rename: + tag: rename_gcp_metrics_commands_usec_per_call_value_to_gcp_redis_commands_usec_per_call_sec_8f1dae96 field: gcp.metrics.commands.usec_per_call.value target_field: gcp.redis.commands.usec_per_call.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_keyspace_avg_ttl_value_to_gcp_redis_keyspace_avg_ttl_sec_4133254e field: gcp.metrics.keyspace.avg_ttl.value target_field: gcp.redis.keyspace.avg_ttl.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_keyspace_keys_value_to_gcp_redis_keyspace_keys_count_a395a830 field: gcp.metrics.keyspace.keys.value target_field: gcp.redis.keyspace.keys.count ignore_missing: true - rename: + tag: rename_gcp_metrics_keyspace_keys_with_expiration_value_to_gcp_redis_keyspace_keys_with_expiration_count_adf94d56 field: gcp.metrics.keyspace.keys_with_expiration.value target_field: gcp.redis.keyspace.keys_with_expiration.count ignore_missing: true - rename: + tag: rename_gcp_metrics_persistence_rdb_bgsave_in_progress_value_to_gcp_redis_persistence_rdb_bgsave_in_progress_06320227 field: gcp.metrics.persistence.rdb.bgsave_in_progress.value target_field: gcp.redis.persistence.rdb.bgsave_in_progress ignore_missing: true - rename: + tag: rename_gcp_metrics_replication_master_slaves_lag_value_to_gcp_redis_replication_master_slaves_lag_sec_749d70f0 field: gcp.metrics.replication.master.slaves.lag.value target_field: gcp.redis.replication.master.slaves.lag.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_replication_master_slaves_offset_value_to_gcp_redis_replication_master_slaves_offset_bytes_34a5f768 field: gcp.metrics.replication.master.slaves.offset.value target_field: gcp.redis.replication.master.slaves.offset.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_replication_master_repl_offset_value_to_gcp_redis_replication_master_repl_offset_bytes_c6657200 field: gcp.metrics.replication.master_repl_offset.value target_field: gcp.redis.replication.master_repl_offset.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_replication_offset_diff_value_to_gcp_redis_replication_offset_diff_bytes_41ebd9d0 field: gcp.metrics.replication.offset_diff.value target_field: gcp.redis.replication.offset_diff.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_replication_role_value_to_gcp_redis_replication_role_0f2d83b7 field: gcp.metrics.replication.role.value target_field: gcp.redis.replication.role ignore_missing: true - rename: + tag: rename_gcp_metrics_server_uptime_value_to_gcp_redis_server_uptime_sec_11979c42 field: gcp.metrics.server.uptime.value target_field: gcp.redis.server.uptime.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_stats_cache_hit_ratio_value_to_gcp_redis_stats_cache_hit_ratio_9d8b2593 field: gcp.metrics.stats.cache_hit_ratio.value target_field: gcp.redis.stats.cache_hit_ratio ignore_missing: true - rename: + tag: rename_gcp_metrics_stats_connections_total_value_to_gcp_redis_stats_connections_total_count_012a022a field: gcp.metrics.stats.connections.total.value target_field: gcp.redis.stats.connections.total.count ignore_missing: true - rename: + tag: rename_gcp_metrics_stats_cpu_utilization_value_to_gcp_redis_stats_cpu_utilization_sec_d47c4cb4 field: gcp.metrics.stats.cpu_utilization.value target_field: gcp.redis.stats.cpu_utilization.sec ignore_missing: true - rename: + tag: rename_gcp_metrics_stats_evicted_keys_value_to_gcp_redis_stats_evicted_keys_count_4b72b464 field: gcp.metrics.stats.evicted_keys.value target_field: gcp.redis.stats.evicted_keys.count ignore_missing: true - rename: + tag: rename_gcp_metrics_stats_expired_keys_value_to_gcp_redis_stats_expired_keys_count_2b67bc38 field: gcp.metrics.stats.expired_keys.value target_field: gcp.redis.stats.expired_keys.count ignore_missing: true - rename: + tag: rename_gcp_metrics_stats_keyspace_hits_value_to_gcp_redis_stats_keyspace_hits_count_e01eaa50 field: gcp.metrics.stats.keyspace_hits.value target_field: gcp.redis.stats.keyspace_hits.count ignore_missing: true - rename: + tag: rename_gcp_metrics_stats_keyspace_misses_value_to_gcp_redis_stats_keyspace_misses_count_d868f948 field: gcp.metrics.stats.keyspace_misses.value target_field: gcp.redis.stats.keyspace_misses.count ignore_missing: true - rename: + tag: rename_gcp_metrics_stats_memory_maxmemory_value_to_gcp_redis_stats_memory_maxmemory_mb_58bfa3a6 field: gcp.metrics.stats.memory.maxmemory.value target_field: gcp.redis.stats.memory.maxmemory.mb ignore_missing: true - rename: + tag: rename_gcp_metrics_stats_memory_system_memory_overload_duration_value_to_gcp_redis_stats_memory_system_memory_overload_duration_us_9b2695e7 field: gcp.metrics.stats.memory.system_memory_overload_duration.value target_field: gcp.redis.stats.memory.system_memory_overload_duration.us ignore_missing: true - rename: + tag: rename_gcp_metrics_stats_memory_system_memory_usage_ratio_value_to_gcp_redis_stats_memory_system_memory_usage_ratio_8bbbcb43 field: gcp.metrics.stats.memory.system_memory_usage_ratio.value target_field: gcp.redis.stats.memory.system_memory_usage_ratio ignore_missing: true - rename: + tag: rename_gcp_metrics_stats_memory_usage_value_to_gcp_redis_stats_memory_usage_bytes_ec7b4fb8 field: gcp.metrics.stats.memory.usage.value target_field: gcp.redis.stats.memory.usage.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_stats_memory_usage_ratio_value_to_gcp_redis_stats_memory_usage_ratio_a9c28a0b field: gcp.metrics.stats.memory.usage_ratio.value target_field: gcp.redis.stats.memory.usage_ratio ignore_missing: true - rename: + tag: rename_gcp_metrics_stats_network_traffic_value_to_gcp_redis_stats_network_traffic_bytes_ccb23b98 field: gcp.metrics.stats.network_traffic.value target_field: gcp.redis.stats.network_traffic.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_stats_pubsub_channels_value_to_gcp_redis_stats_pubsub_channels_count_fd01bfda field: gcp.metrics.stats.pubsub.channels.value target_field: gcp.redis.stats.pubsub.channels.count ignore_missing: true - rename: + tag: rename_gcp_metrics_stats_pubsub_patterns_value_to_gcp_redis_stats_pubsub_patterns_count_0d237f48 field: gcp.metrics.stats.pubsub.patterns.value target_field: gcp.redis.stats.pubsub.patterns.count ignore_missing: true - rename: + tag: rename_gcp_metrics_stats_reject_connections_count_value_to_gcp_redis_stats_reject_connections_count_70e3c940 field: gcp.metrics.stats.reject_connections_count.value target_field: gcp.redis.stats.reject_connections.count ignore_missing: true @@ -137,3 +169,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/gcp/data_stream/storage/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/storage/elasticsearch/ingest_pipeline/default.yml index 827f7075e46..b4a479ba128 100644 --- a/packages/gcp/data_stream/storage/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/storage/elasticsearch/ingest_pipeline/default.yml @@ -2,47 +2,58 @@ description: Pipeline for parsing GCP Storage metrics. processors: - fingerprint: + tag: fingerprint_5c1d1252 fields: - gcp.labels target_field: gcp.labels_fingerprint ignore_missing: true - rename: + tag: rename_gcp_metrics_api_request_count_to_gcp_storage_api_request_count_2a24a5a8 field: gcp.metrics.api.request.count target_field: gcp.storage.api.request.count ignore_missing: true - rename: + tag: rename_gcp_metrics_authz_acl_based_object_access_count_to_gcp_storage_authz_acl_based_object_access_count_087906dc field: gcp.metrics.authz.acl_based_object_access.count target_field: gcp.storage.authz.acl_based_object_access.count ignore_missing: true - rename: + tag: rename_gcp_metrics_authz_acl_operations_count_to_gcp_storage_authz_acl_operations_count_98239a64 field: gcp.metrics.authz.acl_operations.count target_field: gcp.storage.authz.acl_operations.count ignore_missing: true - rename: + tag: rename_gcp_metrics_authz_object_specific_acl_mutation_count_to_gcp_storage_authz_object_specific_acl_mutation_count_85752dbc field: gcp.metrics.authz.object_specific_acl_mutation.count target_field: gcp.storage.authz.object_specific_acl_mutation.count ignore_missing: true - rename: + tag: rename_gcp_metrics_network_received_bytes_to_gcp_storage_network_received_bytes_d344c258 field: gcp.metrics.network.received.bytes target_field: gcp.storage.network.received.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_network_sent_bytes_to_gcp_storage_network_sent_bytes_9915d6a4 field: gcp.metrics.network.sent.bytes target_field: gcp.storage.network.sent.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_storage_object_count_to_gcp_storage_storage_object_count_b2ced43c field: gcp.metrics.storage.object.count target_field: gcp.storage.storage.object.count ignore_missing: true - rename: + tag: rename_gcp_metrics_storage_total_byte_seconds_bytes_to_gcp_storage_storage_total_byte_seconds_bytes_a8fb8d94 field: gcp.metrics.storage.total_byte_seconds.bytes target_field: gcp.storage.storage.total_byte_seconds.bytes ignore_missing: true - rename: + tag: rename_gcp_metrics_storage_total_bytes_to_gcp_storage_storage_total_bytes_64938d14 field: gcp.metrics.storage.total.bytes target_field: gcp.storage.storage.total.bytes ignore_missing: true - remove: + tag: remove_ab6e04c7 field: - gcp.metrics ignore_missing: true @@ -53,3 +64,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/gcp/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml index e76c40c80a1..7ee0e3e58ba 100644 --- a/packages/gcp/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml @@ -3,170 +3,212 @@ description: Pipeline for Google Cloud VPC Flow Logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: 'ctx.event?.original == null' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - community_id: + tag: community_id_46d7956f source_ip: json.jsonPayload.connection.src_ip source_port: json.jsonPayload.connection.src_port destination_ip: json.jsonPayload.connection.dest_ip destination_port: json.jsonPayload.connection.dest_port iana_number: json.jsonPayload.connection.protocol - date: + tag: date_json_timestamp_e111e976 field: json.timestamp timezone: UTC formats: - ISO8601 - set: + tag: set_event_kind_de80643c field: event.kind value: event - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - set: + tag: set_event_id_efd37a25 field: event.id copy_from: json.insertId ignore_empty_value: true ignore_failure: true - set: + tag: set_cloud_provider_62c5e42e field: cloud.provider value: gcp - set: + tag: set_log_logger_01230fab field: log.logger copy_from: json.logName ignore_failure: true - set: + tag: set_destination_address_3f9e1c04 field: destination.address copy_from: json.jsonPayload.connection.dest_ip ignore_failure: true - set: + tag: set_destination_port_48df1199 field: destination.port copy_from: json.jsonPayload.connection.dest_port ignore_failure: true - set: + tag: set_network_iana_number_1bb4e74b field: network.iana_number copy_from: json.jsonPayload.connection.protocol ignore_failure: true - set: + tag: set_source_address_a1ff7ced field: source.address copy_from: json.jsonPayload.connection.src_ip ignore_failure: true - set: + tag: set_source_port_0090c682 field: source.port copy_from: json.jsonPayload.connection.src_port ignore_failure: true - set: + tag: set_source_domain_3b02f5f6 field: source.domain copy_from: json.jsonPayload.src_instance.vm_name ignore_failure: true - set: + tag: set_destination_domain_61437cf3 field: destination.domain copy_from: json.jsonPayload.dest_instance.vm_name ignore_failure: true - set: + tag: set_source_bytes_569b3638 field: source.bytes copy_from: json.jsonPayload.bytes_sent ignore_failure: true - set: + tag: set_source_packets_98215724 field: source.packets copy_from: json.jsonPayload.packets_sent ignore_failure: true - set: + tag: set_event_start_28445d8a field: event.start copy_from: json.jsonPayload.start_time ignore_failure: true - set: + tag: set_event_end_dbfb1714 field: event.end copy_from: json.jsonPayload.end_time ignore_failure: true - set: + tag: set_destination_as_number_44ad410f field: destination.as.number copy_from: json.jsonPayload.dest_location.asn ignore_failure: true - set: + tag: set_destination_geo_continent_name_ab2484fb field: destination.geo.continent_name copy_from: json.jsonPayload.dest_location.continent ignore_failure: true - set: + tag: set_destination_geo_country_name_7918f5b7 field: destination.geo.country_name copy_from: json.jsonPayload.dest_location.country ignore_failure: true - set: + tag: set_destination_geo_region_name_48f7a789 field: destination.geo.region_name copy_from: json.jsonPayload.dest_location.region ignore_failure: true - set: + tag: set_destination_geo_city_name_1739d847 field: destination.geo.city_name copy_from: json.jsonPayload.dest_location.city ignore_failure: true - set: + tag: set_source_as_number_2d01b872 field: source.as.number copy_from: json.jsonPayload.src_location.asn ignore_failure: true - set: + tag: set_source_geo_continent_name_440a3660 field: source.geo.continent_name copy_from: json.jsonPayload.src_location.continent ignore_failure: true - set: + tag: set_source_geo_country_name_f15fd6c8 field: source.geo.country_name copy_from: json.jsonPayload.src_location.country ignore_failure: true - set: + tag: set_source_geo_region_name_453025f4 field: source.geo.region_name copy_from: json.jsonPayload.src_location.region ignore_failure: true - set: + tag: set_source_geo_city_name_653753c4 field: source.geo.city_name copy_from: json.jsonPayload.src_location.city ignore_failure: true - set: + tag: set_gcp_destination_instance_73b610f1 field: gcp.destination.instance copy_from: json.jsonPayload.dest_instance ignore_failure: true - set: + tag: set_gcp_destination_vpc_665d2e57 field: gcp.destination.vpc copy_from: json.jsonPayload.dest_vpc ignore_failure: true - set: + tag: set_gcp_source_instance_77b689be field: gcp.source.instance copy_from: json.jsonPayload.src_instance ignore_failure: true - set: + tag: set_gcp_source_vpc_e907735e field: gcp.source.vpc copy_from: json.jsonPayload.src_vpc ignore_failure: true - convert: + tag: convert_json_jsonPayload_rtt_msec_to_json_jsonPayload_rtt_ms_a104658a field: json.jsonPayload.rtt_msec target_field: json.jsonPayload.rtt.ms type: long ignore_missing: true - rename: + tag: rename_json_jsonPayload_to_gcp_vpcflow_f7a42ae5 field: json.jsonPayload target_field: gcp.vpcflow ignore_missing: true - convert: + tag: convert_source_bytes_e0c47eeb field: source.bytes type: long ignore_missing: true - convert: + tag: convert_source_packets_fc56b0f9 field: source.packets type: long ignore_missing: true - convert: + tag: convert_network_iana_number_b5dcb911 field: network.iana_number type: string ignore_missing: true - script: + tag: script_aaef8fa9 lang: painless ignore_failure: true if: ctx?.network?.iana_number != null @@ -196,40 +238,47 @@ processors: ctx.network.transport = 'sctp'; } - set: + tag: set_source_ip_1bfe096a field: source.ip copy_from: source.address ignore_failure: true if: ctx.source?.address != null - set: + tag: set_destination_ip_6e761801 field: destination.ip copy_from: destination.address ignore_failure: true if: ctx.destination?.address != null - convert: + tag: convert_gcp_source_instance_project_id_to_cloud_project_id_43fdab19 field: gcp.source.instance.project_id target_field: cloud.project.id type: string ignore_missing: true if: ctx?.gcp?.vpcflow?.reporter == "DEST" - convert: + tag: convert_gcp_source_instance_vm_name_to_cloud_instance_name_4f098fe7 field: gcp.source.instance.vm_name target_field: cloud.instance.name type: string ignore_missing: true if: ctx?.gcp?.vpcflow?.reporter == "DEST" - convert: + tag: convert_gcp_source_instance_region_to_cloud_region_140a3aea field: gcp.source.instance.region target_field: cloud.region type: string ignore_missing: true if: ctx?.gcp?.vpcflow?.reporter == "DEST" - convert: + tag: convert_gcp_source_instance_zone_to_cloud_availability_zone_bc3e960c field: gcp.source.instance.zone target_field: cloud.availability_zone type: string ignore_missing: true if: ctx?.gcp?.vpcflow?.reporter == "DEST" - convert: + tag: convert_gcp_source_vpc_subnetwork_name_to_network_name_f0abac9d field: gcp.source.vpc.subnetwork_name target_field: network.name type: string @@ -237,30 +286,35 @@ processors: ignore_failure: true if: ctx?.gcp?.vpcflow?.reporter == "DEST" - convert: + tag: convert_gcp_destination_instance_project_id_to_cloud_project_id_930d8780 field: gcp.destination.instance.project_id target_field: cloud.project.id type: string ignore_missing: true if: ctx?.gcp?.vpcflow?.reporter == "SRC" - convert: + tag: convert_gcp_destination_instance_vm_name_to_cloud_instance_name_fb68460a field: gcp.destination.instance.vm_name target_field: cloud.instance.name type: string ignore_missing: true if: ctx?.gcp?.vpcflow?.reporter == "SRC" - convert: + tag: convert_gcp_destination_instance_region_to_cloud_region_f4c2df7d field: gcp.destination.instance.region target_field: cloud.region type: string ignore_missing: true if: ctx?.gcp?.vpcflow?.reporter == "SRC" - convert: + tag: convert_gcp_destination_instance_zone_to_cloud_availability_zone_613f72c1 field: gcp.destination.instance.zone target_field: cloud.availability_zone type: string ignore_missing: true if: ctx?.gcp?.vpcflow?.reporter == "SRC" - convert: + tag: convert_gcp_destination_vpc_subnetwork_name_to_network_name_ed7bedc6 field: gcp.destination.vpc.subnetwork_name target_field: network.name type: string @@ -268,60 +322,73 @@ processors: ignore_failure: true if: ctx?.gcp?.vpcflow?.reporter == "SRC" - convert: + tag: convert_source_bytes_to_network_bytes_298da900 field: source.bytes type: long target_field: network.bytes ignore_missing: true - convert: + tag: convert_source_packets_to_network_packets_adc0c3b4 field: source.packets type: long target_field: network.packets ignore_missing: true - set: + tag: set_network_direction_58050b34 field: network.direction value: internal if: ctx?.gcp?.source?.instance != null && ctx?.gcp?.destination?.instance != null - set: + tag: set_network_direction_d46c6d3b field: network.direction value: outbound if: ctx?.gcp?.source?.instance != null && ctx?.gcp?.destination?.instance == null - set: + tag: set_network_direction_08c4e8ee field: network.direction value: inbound if: ctx?.gcp?.source?.instance == null && ctx?.gcp?.destination?.instance != null - set: + tag: set_network_direction_f672f5c9 field: network.direction value: unknown if: ctx?.network?.direction == null - set: + tag: set_network_type_409f99d1 field: network.type value: ipv4 - if: ctx?.source?.ip != null && ctx?.source?.ip.contains(".") + if: ctx?.source?.ip != null && ctx?.source?.ip.contains(".") - set: + tag: set_network_type_3a24ca9a field: network.type value: ipv6 if: ctx?.source?.ip != null && !ctx?.source?.ip.contains(".") - append: + tag: append_related_ip_c40a2e39 field: related.ip value: "{{source.ip}}" allow_duplicates: false if: ctx?.source?.ip != null && ctx?.source?.ip != "" - append: + tag: append_related_ip_b730887a field: related.ip value: "{{destination.ip}}" allow_duplicates: false if: ctx?.destination?.ip != null && ctx?.destination?.ip != "" # IP Geolocation Lookup - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true # IP Autonomous System (AS) Lookup - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -330,6 +397,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -338,26 +406,32 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - rename: + tag: rename_json_to_gcp_vpcflow_flattened_8e3f5d9b field: json target_field: gcp.vpcflow.flattened if: ctx.json != null && ctx._conf?.keep_json == true - remove: + tag: remove_19bf017b field: - gcp.destination.instance.vm_name - gcp.source.instance.vm_name @@ -400,3 +474,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/gcp/manifest.yml b/packages/gcp/manifest.yml index 94fd86121b8..49f03253580 100644 --- a/packages/gcp/manifest.yml +++ b/packages/gcp/manifest.yml @@ -1,6 +1,6 @@ name: gcp title: Google Cloud Platform -version: "2.50.2" +version: "2.50.3" description: Collect logs and metrics from Google Cloud Platform with Elastic Agent. type: integration icons: diff --git a/packages/github/changelog.yml b/packages/github/changelog.yml index ca5dad09694..8e46f8ef914 100644 --- a/packages/github/changelog.yml +++ b/packages/github/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.25.3" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "2.25.2" changes: - description: Add `event.action` and `event.reason` to the `issues` data stream. diff --git a/packages/github/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/github/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 9af6760ca47..1d0bb3e186f 100644 --- a/packages/github/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/github/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -2,9 +2,11 @@ description: Pipeline for parsing GitHub audit logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - append: + tag: append_event_category_e1ee9a8c field: event.category value: - configuration @@ -23,17 +25,21 @@ processors: description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - fail: + tag: fail_74459f8f if: "!(ctx.json instanceof Map)" message: Missing JSON object - fingerprint: + tag: fingerprint_0a8a3679 fields: - json._document_id target_field: "_id" ignore_missing: true - rename: + tag: rename_json_to_github_2879b127 field: json target_field: github ignore_missing: true @@ -47,8 +53,10 @@ processors: if: ctx.github?.created_at != null on_failure: - remove: + tag: remove_github_created_at_010a80b6 field: github.created_at - append: + tag: append_error_message_040fdc9f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -61,39 +69,49 @@ processors: if: ctx.github?.get('@timestamp') != null on_failure: - remove: + tag: remove_github_timestamp_364ffc55 field: github.@timestamp - append: + tag: append_error_message_35a6b42a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_github__document_id_to_event_id_26a6bbc2 field: github._document_id target_field: event.id ignore_missing: true - rename: + tag: rename_github_action_to_event_action_05524b39 field: github.action target_field: event.action ignore_missing: true - set: + tag: set_event_kind_0d477994 field: event.kind value: alert if: ctx.event?.action?.equalsIgnoreCase('code_scanning.alert_created') == true || ctx.event?.action?.equalsIgnoreCase('secret_scanning_alert.create') == true - set: + tag: set_event_kind_31b9e9ac field: event.kind value: event if: ctx.event?.kind == null - rename: + tag: rename_github_actor_to_user_name_8172ce88 field: github.actor target_field: user.name ignore_missing: true - append: + tag: append_related_user_c036eec2 field: related.user value: "{{{user.name}}}" if: ctx.user?.name != null - rename: + tag: rename_github_user_to_user_target_name_d847b55f field: github.user target_field: user.target.name ignore_missing: true - append: + tag: append_related_user_9680e737 field: related.user value: "{{{user.target.name}}}" if: ctx.user?.target?.name != null @@ -105,45 +123,55 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_github_repository_public_6045b964 field: github.repository_public - append: + tag: append_error_message_4e2d87bd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_github_data_team_to_github_team_a4f357ba field: github.data.team target_field: github.team ignore_missing: true if: ctx.github?.team == null - set: + tag: set_group_name_aaad2615 field: group.name copy_from: github.team ignore_empty_value: true if: ctx.event?.action.startsWith("team.") - set: + tag: set_user_target_group_name_c2ed003d field: user.target.group.name copy_from: github.team ignore_empty_value: true if: ctx.event?.action?.startsWith("team.") == true && ctx.user?.target?.name != null - set: + tag: set_group_name_42257061 field: group.name copy_from: github.org ignore_empty_value: true if: ctx.event?.action.startsWith("org.") - set: + tag: set_user_target_group_name_139e1ca5 field: user.target.group.name copy_from: github.org ignore_empty_value: true if: ctx.event?.action?.startsWith("org.") == true && ctx.user?.target?.name != null - rename: + tag: rename_github_data_old_user_to_user_target_group_name_9ef1cbba field: github.data.old_user target_field: user.target.group.name ignore_missing: true - rename: + tag: rename_github_data_old_user_to_user_target_group_name_6068f8fa field: github.data.old_user target_field: user.target.group.name ignore_missing: true if: ctx.user?.target?.group?.name == null - rename: + tag: rename_github_actor_location_country_code_to_client_geo_country_iso_code_5d630bf4 field: github.actor_location.country_code target_field: client.geo.country_iso_code ignore_missing: true @@ -156,18 +184,23 @@ processors: if: ctx.github?.actor_ip != null && ctx.github.actor_ip != '' on_failure: - remove: + tag: remove_github_actor_ip_e7305734 field: github.actor_ip - append: + tag: append_error_message_95a3de54 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: + tag: append_related_ip_a83ff53c field: related.ip value: '{{{github.actor_ip}}}' if: ctx.github?.actor_ip != null - user_agent: + tag: user_agent_github_user_agent_bc60ddd3 field: github.user_agent ignore_missing: true - grok: + tag: grok_event_action_e7945a25 field: event.action ignore_missing: true patterns: @@ -175,24 +208,29 @@ processors: pattern_definitions: GH_CAT: '[a-z_]+' - append: + tag: append_event_category_5634efe9 field: event.category value: iam if: '["team","org"].contains(ctx.github.category)' - append: + tag: append_event_type_18178cc4 field: event.type value: - group - user if: '["team","org"].contains(ctx.github.category)' - append: + tag: append_event_type_b4084fd9 field: event.type value: creation if: ctx._temp?.action?.contains("create") == true || ctx._temp?.action?.contains("add") == true - append: + tag: append_event_type_57265258 field: event.type value: deletion if: ctx._temp?.action?.contains("delete") == true || ctx._temp?.action?.contains("remove") == true - append: + tag: append_event_type_65cf1e89 field: event.type value: change if: ctx.event?.type == null || ctx.event.type.size() == 0 @@ -203,8 +241,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_github_active_87fa9c9b field: github.active - append: + tag: append_error_message_dfb9ec0e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -214,8 +254,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_github_actor_is_bot_14e17c63 field: github.actor_is_bot - append: + tag: append_error_message_57660dd0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -225,8 +267,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_github_audit_log_stream_enabled_5d6f3d9b field: github.audit_log_stream_enabled - append: + tag: append_error_message_2495c0e9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -266,7 +310,7 @@ processors: ignore_missing: true - set: field: user.target.id - tag: set_user_target_id + tag: set_user_target_id_29ab9270 copy_from: github.user_id ignore_empty_value: true - append: @@ -303,8 +347,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_github_transport_protocol_78168b0b field: github.transport_protocol - append: + tag: append_error_message_0e3a2d82 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -326,8 +372,10 @@ processors: if: ctx.github?.data?.started_at != null && ctx.github.data.started_at != '' on_failure: - remove: + tag: remove_github_data_started_at_07afb8dc field: github.data.started_at - append: + tag: append_error_message_2a08829e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -352,8 +400,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_github_public_repo_d5b455d1 field: github.public_repo - append: + tag: append_error_message_2824ad42 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -364,8 +414,10 @@ processors: if: ctx.github?.actor_location?.ip != '' on_failure: - remove: + tag: remove_github_actor_location_ip_d524d7b7 field: github.actor_location.ip - append: + tag: append_error_message_5b3e778e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -419,6 +471,7 @@ processors: value: '{{{github.data.head_sha}}}' allow_duplicates: false - remove: + tag: remove_be1d7231 field: - _temp - github.created_at @@ -432,6 +485,7 @@ processors: if: ctx.aws?.s3?.bucket != null && ctx.aws.s3.object != null ignore_missing: true - script: + tag: script_4d6993cb lang: painless description: This script processor iterates over the whole document to remove fields with null values. source: | diff --git a/packages/github/data_stream/code_scanning/elasticsearch/ingest_pipeline/default.yml b/packages/github/data_stream/code_scanning/elasticsearch/ingest_pipeline/default.yml index 6975be75827..4c1925698fb 100644 --- a/packages/github/data_stream/code_scanning/elasticsearch/ingest_pipeline/default.yml +++ b/packages/github/data_stream/code_scanning/elasticsearch/ingest_pipeline/default.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing GitHub audit logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - rename: @@ -18,23 +19,29 @@ processors: description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null - json: + tag: json_event_original_to_github_code_scanning_7a97be3c field: event.original target_field: github.code_scanning - fail: + tag: fail_0524996d if: "!(ctx.github.code_scanning instanceof Map)" message: Missing JSON object - set: + tag: set_event_kind_39295792 field: event.kind value: alert - append: + tag: append_event_type_449ae2bb field: event.type value: creation if: ctx.github?.code_scanning?.fixed_at == null && ctx.github?.code_scanning?.dismissed_at == null - append: + tag: append_event_type_7b5dee62 field: event.type value: deletion if: ctx.github?.code_scanning?.fixed_at != null || ctx.github?.code_scanning?.dismissed_at != null - date: + tag: date_github_code_scanning_created_at_to_event_created_63a63689 field: github.code_scanning.created_at formats: - ISO8601 @@ -42,6 +49,7 @@ processors: target_field: "event.created" if: ctx.github.code_scanning.created_at != null - date: + tag: date_github_code_scanning_created_at_to_timestamp_feaeecef field: github.code_scanning.created_at formats: - ISO8601 @@ -49,6 +57,7 @@ processors: target_field: "@timestamp" if: ctx.github.code_scanning.created_at != null - date: + tag: date_github_code_scanning_updated_at_to_timestamp_d54416b5 field: github.code_scanning.updated_at formats: - ISO8601 @@ -56,6 +65,7 @@ processors: target_field: "@timestamp" if: ctx.github.code_scanning.updated_at != null - date: + tag: date_github_code_scanning_dismissed_at_to_github_code_scanning_dismissed_at_46798104 field: github.code_scanning.dismissed_at formats: - ISO8601 @@ -63,93 +73,115 @@ processors: target_field: github.code_scanning.dismissed_at if: ctx.github?.code_scanning?.dismissed_at != null - rename: + tag: rename_github_code_scanning_repository_to__temp_fa3b2ed5 target_field: _temp field: github.code_scanning.repository ignore_missing: true - rename: + tag: rename__temp_id_to_github_repository_id_327ad07c field: _temp.id target_field: github.repository.id ignore_missing: true - rename: + tag: rename__temp_name_to_github_repository_name_2866cc74 field: _temp.name target_field: github.repository.name ignore_missing: true - rename: + tag: rename__temp_full_name_to_github_repository_full_name_e875d03e field: _temp.full_name target_field: github.repository.full_name ignore_missing: true - rename: + tag: rename__temp_private_to_github_repository_private_8af96072 field: _temp.private target_field: github.repository.private ignore_missing: true - rename: + tag: rename__temp_html_url_to_github_repository_html_url_28d63710 field: _temp.html_url target_field: github.repository.html_url ignore_missing: true - rename: + tag: rename__temp_url_to_github_repository_url_6a22e32e field: _temp.url target_field: github.repository.url ignore_missing: true - rename: + tag: rename__temp_description_to_github_repository_description_bb25b568 field: _temp.description target_field: github.repository.description ignore_missing: true - rename: + tag: rename__temp_fork_to_github_repository_fork_f89db4d0 field: _temp.fork target_field: github.repository.fork ignore_missing: true - rename: + tag: rename__temp_owner_login_to_github_repository_owner_login_65889624 field: _temp.owner.login target_field: github.repository.owner.login ignore_missing: true - rename: + tag: rename__temp_owner_id_to_github_repository_owner_id_7ec1ab08 field: _temp.owner.id target_field: github.repository.owner.id ignore_missing: true - rename: + tag: rename__temp_owner_url_to_github_repository_owner_url_f442f74c field: _temp.owner.url target_field: github.repository.owner.url ignore_missing: true - rename: + tag: rename__temp_owner_html_url_to_github_repository_owner_html_url_6d19c164 field: _temp.owner.html_url target_field: github.repository.owner.html_url ignore_missing: true - rename: + tag: rename__temp_owner_type_to_github_repository_owner_type_7e1b8774 field: _temp.owner.type target_field: github.repository.owner.type ignore_missing: true - rename: + tag: rename__temp_owner_site_admin_to_github_repository_owner_site_admin_fa5ae018 field: _temp.owner.site_admin target_field: github.repository.owner.site_admin ignore_missing: true - dissect: + tag: dissect_github_code_scanning_url_1e82b0ca field: github.code_scanning.url pattern: "https://api.github.com/repos/%{_temp.owner}/%{_temp.repository}/code-scanning/alerts/%{_temp.number}" ignore_missing: true - set: + tag: set_github_repository_html_url_ec5e1981 field: github.repository.html_url value: "https://github.com/{{{_temp.owner}}}/{{{_temp.repository}}}" if: ctx.github.repository?.html_url == null && ctx._temp?.owner != null && ctx._temp?.repository != null - set: + tag: set_github_repository_url_f0b322a3 field: github.repository.url value: "https://api.github.com/repos/{{{_temp.owner}}}/{{{_temp.repository}}}" if: ctx.github.repository?.url == null && ctx._temp?.owner != null && ctx._temp?.repository != null - rename: + tag: rename__temp_repository_to_github_repository_name_c2e74f39 field: _temp.repository target_field: github.repository.name if: ctx.github.repository?.name == null ignore_missing: true - rename: + tag: rename__temp_owner_to_github_repository_owner_login_f9ff210d field: _temp.owner target_field: github.repository.owner.login if: ctx.github.repository?.owner?.login == null ignore_missing: true - rename: + tag: rename__temp_number_to_github_code_scanning_number_b36ddfb3 field: _temp.number target_field: github.code_scanning.number if: ctx.github.code_scanning.number == null ignore_missing: true - fingerprint: + tag: fingerprint_9319bf6b fields: - github.repository.owner.login - github.repository.name @@ -160,47 +192,59 @@ processors: target_field: "_id" ignore_missing: true - lowercase: + tag: lowercase_github_code_scanning_state_06b21ee0 field: github.code_scanning.state ignore_missing: true - lowercase: + tag: lowercase_github_code_scanning_rule_security_severity_level_2822e234 field: github.code_scanning.rule.security_severity_level ignore_missing: true - rename: + tag: rename_github_code_scanning_dismissed_by_to__temp_dismissed_by_fa70a3d5 target_field: _temp.dismissed_by field: github.code_scanning.dismissed_by ignore_missing: true - remove: + tag: remove_github_code_scanning_dismissed_by_0b57f54f field: github.code_scanning.dismissed_by ignore_missing: true - rename: + tag: rename__temp_dismissed_by_login_to_github_code_scanning_dismissed_by_login_5b819cb5 field: _temp.dismissed_by.login target_field: github.code_scanning.dismissed_by.login ignore_missing: true - rename: + tag: rename__temp_dismissed_by_id_to_github_code_scanning_dismissed_by_id_0d9039a7 field: _temp.dismissed_by.id target_field: github.code_scanning.dismissed_by.id ignore_missing: true - rename: + tag: rename__temp_dismissed_by_url_to_github_code_scanning_dismissed_by_url_a408e6d5 field: _temp.dismissed_by.url target_field: github.code_scanning.dismissed_by.url ignore_missing: true - rename: + tag: rename__temp_dismissed_by_html_url_to_github_code_scanning_dismissed_by_html_url_66ea2cdb field: _temp.dismissed_by.html_url target_field: github.code_scanning.dismissed_by.html_url ignore_missing: true - rename: + tag: rename__temp_dismissed_by_type_to_github_code_scanning_dismissed_by_type_567c11f3 field: _temp.dismissed_by.type target_field: github.code_scanning.dismissed_by.type ignore_missing: true - rename: + tag: rename__temp_dismissed_by_site_admin_to_github_code_scanning_dismissed_by_site_admin_45f098ff field: _temp.dismissed_by.site_admin target_field: github.code_scanning.dismissed_by.site_admin ignore_missing: true - rename: + tag: rename_github_code_scanning_most_recent_instance_message_text_to_message_8c8d19e0 field: github.code_scanning.most_recent_instance.message.text target_field: message ignore_missing: true - script: + tag: script_f715f9a4 lang: painless description: Calculate time taken to resolve the alerts if: ctx.github.code_scanning.fixed_at != null || ctx.github.code_scanning.dismissed_at != null @@ -227,18 +271,22 @@ processors: # Rule ECS Fields # ################### - rename: + tag: rename_github_code_scanning_rule_id_to_rule_id_c69ecc48 field: github.code_scanning.rule.id target_field: rule.id if: ctx.github.code_scanning.rule?.id != null - rename: + tag: rename_github_code_scanning_rule_name_to_rule_name_95f8f72e field: github.code_scanning.rule.name target_field: rule.name if: ctx.github.code_scanning.rule?.name != null - rename: + tag: rename_github_code_scanning_rule_description_to_rule_description_ce15d0a7 field: github.code_scanning.rule.description target_field: rule.description if: ctx.github.code_scanning.rule?.description != null - foreach: + tag: foreach_github_code_scanning_rule_tags_abb933eb field: "github.code_scanning.rule.tags" processor: append: @@ -247,16 +295,19 @@ processors: ignore_missing: true if: ctx.github.code_scanning.rule?.tags != null - remove: + tag: remove_github_code_scanning_rule_tags_f195e1c7 field: github.code_scanning.rule.tags ignore_missing: true ########### # Cleanup # ########### - remove: + tag: remove_1f8c5875 field: - _temp ignore_missing: true - script: + tag: script_4d6993cb lang: painless description: This script processor iterates over the whole document to remove fields with null values. source: | diff --git a/packages/github/data_stream/dependabot/elasticsearch/ingest_pipeline/default.yml b/packages/github/data_stream/dependabot/elasticsearch/ingest_pipeline/default.yml index f0d4d89c5c5..82072120d2f 100644 --- a/packages/github/data_stream/dependabot/elasticsearch/ingest_pipeline/default.yml +++ b/packages/github/data_stream/dependabot/elasticsearch/ingest_pipeline/default.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing GitHub audit logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: "8.11.0" - rename: @@ -18,22 +19,28 @@ processors: description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null - json: + tag: json_event_original_to_github_dependabot_a52b24cb field: event.original target_field: github.dependabot - fail: + tag: fail_1e9d0cfa if: "!(ctx.github.dependabot instanceof Map)" message: Missing JSON object - set: + tag: set_event_kind_39295792 field: event.kind value: alert - set: + tag: set__temp_updated_at_b3c03583 field: _temp.updated_at value: "{{{_ingest.timestamp}}}" - rename: + tag: rename_github_dependabot_createdAt_to_github_dependabot_created_at_dfab9887 field: github.dependabot.createdAt target_field: github.dependabot.created_at ignore_missing: true - date: + tag: date_github_dependabot_created_at_to_event_created_ce5d5d95 field: github.dependabot.created_at formats: - ISO8601 @@ -41,30 +48,37 @@ processors: target_field: "event.created" if: ctx.github.dependabot.created_at != null - rename: + tag: rename_github_dependabot_dependabotUpdate_to_github_dependabot_dependabot_update_91dddec9 field: github.dependabot.dependabotUpdate target_field: github.dependabot.dependabot_update ignore_missing: true - rename: + tag: rename_github_dependabot_dependabot_update_error_errorType_to_github_dependabot_dependabot_update_error_error_type_028d84bb field: github.dependabot.dependabot_update.error.errorType target_field: github.dependabot.dependabot_update.error.error_type ignore_missing: true - rename: + tag: rename_github_dependabot_dependencyScope_to_github_dependabot_dependency_scope_520708a7 field: github.dependabot.dependencyScope target_field: github.dependabot.dependency_scope ignore_missing: true - rename: + tag: rename_github_dependabot_dismissReason_to_github_dependabot_dismiss_reason_ec6d7b4f field: github.dependabot.dismissReason target_field: github.dependabot.dismiss_reason ignore_missing: true - rename: + tag: rename_github_dependabot_dismissedAt_to_github_dependabot_dismissed_at_7993e785 field: github.dependabot.dismissedAt target_field: github.dependabot.dismissed_at ignore_missing: true - rename: + tag: rename_github_dependabot_fixedAt_to_github_dependabot_fixed_at_6ce642bb field: github.dependabot.fixedAt target_field: github.dependabot.fixed_at ignore_missing: true - date: + tag: date_github_dependabot_created_at_to_timestamp_6c3f387b field: github.dependabot.created_at formats: - ISO8601 @@ -72,6 +86,7 @@ processors: target_field: "@timestamp" if: ctx.github.dependabot.created_at != null - date: + tag: date_github_dependabot_dismissed_at_to_timestamp_4d46df53 field: github.dependabot.dismissed_at formats: - ISO8601 @@ -79,6 +94,7 @@ processors: target_field: "@timestamp" if: ctx.github.dependabot.dismissed_at != null - date: + tag: date_github_dependabot_fixed_at_to_timestamp_a9249217 field: github.dependabot.fixed_at formats: - ISO8601 @@ -86,53 +102,66 @@ processors: target_field: "@timestamp" if: ctx.github.dependabot.fixed_at != null - append: + tag: append_event_type_8cb07c5b field: event.type value: creation if: ctx.github?.dependabot?.fixed_at == null && ctx.github?.dependabot?.dismissed_at == null - append: + tag: append_event_type_d62d79ba field: event.type value: deletion if: ctx.github?.dependabot?.fixed_at != null || ctx.github?.dependabot?.dismissed_at != null - rename: + tag: rename_github_dependabot_repository_isInOrganization_to_github_dependabot_repository_is_in_organization_0bce7d18 field: github.dependabot.repository.isInOrganization target_field: github.dependabot.repository.is_in_organization ignore_missing: true - rename: + tag: rename_github_dependabot_repository_isPrivate_to_github_dependabot_repository_private_de1397fc field: github.dependabot.repository.isPrivate target_field: github.dependabot.repository.private ignore_missing: true - rename: + tag: rename_github_dependabot_securityAdvisory_to_github_dependabot_security_advisory_146907e9 field: github.dependabot.securityAdvisory target_field: github.dependabot.security_advisory ignore_missing: true - rename: + tag: rename_github_dependabot_security_advisory_cvss_vectorString_to_github_dependabot_security_advisory_cvss_vector_string_2985e277 field: github.dependabot.security_advisory.cvss.vectorString target_field: github.dependabot.security_advisory.cvss.vector_string ignore_missing: true - rename: + tag: rename_github_dependabot_security_advisory_ghsaId_to_github_dependabot_security_advisory_ghsa_id_1d6f6805 field: github.dependabot.security_advisory.ghsaId target_field: github.dependabot.security_advisory.ghsa_id ignore_missing: true - rename: + tag: rename_github_dependabot_security_advisory_publishedAt_to_github_dependabot_security_advisory_published_at_e9fdb73b field: github.dependabot.security_advisory.publishedAt target_field: github.dependabot.security_advisory.published_at ignore_missing: true - rename: + tag: rename_github_dependabot_security_advisory_updatedAt_to_github_dependabot_security_advisory_updated_at_e27aeab5 field: github.dependabot.security_advisory.updatedAt target_field: github.dependabot.security_advisory.updated_at ignore_missing: true - set: + tag: set__temp_cwes_2c9da423 field: _temp.cwes copy_from: github.dependabot.security_advisory.cwes.nodes if: ctx.github.dependabot.security_advisory?.cwes?.nodes != null - remove: + tag: remove_github_dependabot_security_advisory_cwes_ba600c65 field: github.dependabot.security_advisory.cwes ignore_missing: true - set: + tag: set_github_dependabot_security_advisory_cwes_2ac54574 field: github.dependabot.security_advisory.cwes copy_from: _temp.cwes if: ctx._temp?.cwes != null - foreach: + tag: foreach_github_dependabot_security_advisory_cwes_a8a78acc field: github.dependabot.security_advisory.cwes if: ctx.github?.dependabot?.security_advisory?.cwes instanceof List ignore_failure: true @@ -143,30 +172,37 @@ processors: target_field: _ingest._value.cwe_id ignore_missing: true - rename: + tag: rename_github_dependabot_securityVulnerability_to_github_dependabot_security_vulnerability_4ce3d9a1 field: github.dependabot.securityVulnerability target_field: github.dependabot.security_vulnerability ignore_missing: true - rename: + tag: rename_github_dependabot_security_vulnerability_firstPatchedVersion_to_github_dependabot_security_vulnerability_first_patched_version_0099fe22 field: github.dependabot.security_vulnerability.firstPatchedVersion target_field: github.dependabot.security_vulnerability.first_patched_version ignore_missing: true - rename: + tag: rename_github_dependabot_security_vulnerability_updatedAt_to_github_dependabot_security_vulnerability_updated_at_467422f1 field: github.dependabot.security_vulnerability.updatedAt target_field: github.dependabot.security_vulnerability.updated_at ignore_missing: true - rename: + tag: rename_github_dependabot_security_vulnerability_vulnerableVersionRange_to_github_dependabot_security_vulnerability_vulnerable_version_range_e6bc3aac field: github.dependabot.security_vulnerability.vulnerableVersionRange target_field: github.dependabot.security_vulnerability.vulnerable_version_range ignore_missing: true - rename: + tag: rename_github_dependabot_vulnerableManifestFilename_to_github_dependabot_vulnerable_manifest_filename_f1209858 field: github.dependabot.vulnerableManifestFilename target_field: github.dependabot.vulnerable_manifest_filename ignore_missing: true - rename: + tag: rename_github_dependabot_vulnerableManifestPath_to_github_dependabot_vulnerable_manifest_path_774ac49c field: github.dependabot.vulnerableManifestPath target_field: github.dependabot.vulnerable_manifest_path ignore_missing: true - rename: + tag: rename_github_dependabot_vulnerableRequirements_to_github_dependabot_vulnerable_requirements_7858ba9b field: github.dependabot.vulnerableRequirements target_field: github.dependabot.vulnerable_requirements ignore_missing: true @@ -174,13 +210,16 @@ processors: # Vulnerability ECS Fields # ############################ - set: + tag: set_vulnerability_classification_8b34b171 field: vulnerability.classification value: "CVSS" - rename: + tag: rename_github_dependabot_security_advisory_description_to_vulnerability_description_38a0cb37 target_field: vulnerability.description field: github.dependabot.security_advisory.description ignore_missing: true - script: + tag: script_637353c2 lang: painless description: Set vulnerability.enumeration and vulnerability.id ECS fields. Defaults to GHSA if: ctx.github.dependabot.security_advisory?.identifiers != null @@ -198,6 +237,7 @@ processors: ctx.vulnerability.enumeration = enumeration; ctx.vulnerability.id = id; - script: + tag: script_ab0150c5 lang: painless description: Set vulnerability.reference ECS field if: ctx.github.dependabot.security_advisory?.references != null @@ -209,19 +249,24 @@ processors: } ctx.vulnerability.reference = references; - remove: + tag: remove_github_dependabot_security_advisory_references_4cfb9e25 field: github.dependabot.security_advisory.references ignore_missing: true - set: + tag: set_vulnerability_scanner_vendor_743a33e3 field: vulnerability.scanner.vendor value: "Github" - convert: + tag: convert_github_dependabot_security_advisory_cvss_score_87a16248 field: github.dependabot.security_advisory.cvss.score type: float - rename: + tag: rename_github_dependabot_security_advisory_cvss_score_to_vulnerability_score_base_568e6f7b target_field: vulnerability.score.base field: github.dependabot.security_advisory.cvss.score ignore_missing: true - kv: + tag: kv_github_dependabot_security_advisory_cvss_vector_string_to__temp_score_version_e89cab09 field: github.dependabot.security_advisory.cvss.vector_string target_field: "_temp.score.version" field_split: "/" @@ -229,10 +274,12 @@ processors: include_keys: ["CVSS"] ignore_missing: true - rename: + tag: rename__temp_score_version_CVSS_to_vulnerability_score_version_e75a2f2a target_field: vulnerability.score.version field: _temp.score.version.CVSS ignore_missing: true - rename: + tag: rename_github_dependabot_security_vulnerability_severity_to_vulnerability_severity_5230e8b2 field: github.dependabot.security_vulnerability.severity target_field: vulnerability.severity ignore_missing: true @@ -240,18 +287,22 @@ processors: # Event ECS Fields # ############################ - set: + tag: set_event_start_0f086898 field: "event.start" copy_from: github.dependabot.created_at if: ctx.github.dependabot.created_at != null - set: + tag: set_event_end_f390ee2b field: "event.end" copy_from: github.dependabot.fixed_at if: ctx.github.dependabot.fixed_at != null - set: + tag: set_event_end_cbb283a7 field: "event.end" copy_from: github.dependabot.dismissed_at if: ctx.event?.end == null && ctx.github.dependabot.dismissed_at != null - script: + tag: script_363d2d06 lang: painless description: Calculate time taken to resolve the alerts if: ctx.event?.start != null && ctx.event?.end != null @@ -261,16 +312,20 @@ processors: # For GitHub Overview Dashboard # ################################# - lowercase: + tag: lowercase_github_dependabot_state_503175bf field: github.dependabot.state ignore_missing: true - lowercase: + tag: lowercase_vulnerability_severity_2092d4dc field: vulnerability.severity ignore_missing: true - rename: + tag: rename_github_dependabot_repository_to_github_repository_bc24b0a4 field: github.dependabot.repository target_field: github.repository ignore_missing: true - fingerprint: + tag: fingerprint_eca23086 fields: - github.repository.owner.login - github.repository.name @@ -284,10 +339,12 @@ processors: # Cleanup # ########### - remove: + tag: remove_1f8c5875 field: - _temp ignore_missing: true - script: + tag: script_4d6993cb lang: painless description: This script processor iterates over the whole document to remove fields with null values. source: | diff --git a/packages/github/data_stream/issues/elasticsearch/ingest_pipeline/default.yml b/packages/github/data_stream/issues/elasticsearch/ingest_pipeline/default.yml index 5123af5a37b..1530dca498e 100644 --- a/packages/github/data_stream/issues/elasticsearch/ingest_pipeline/default.yml +++ b/packages/github/data_stream/issues/elasticsearch/ingest_pipeline/default.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing GitHub audit logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - rename: @@ -18,28 +19,34 @@ processors: description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null - json: + tag: json_event_original_to_github_issues_b6b557e5 field: event.original target_field: github.issues - fail: + tag: fail_2df5ab84 if: "!(ctx.github.issues instanceof Map)" message: Missing JSON object - set: + tag: set_event_kind_de80643c field: event.kind value: event # The API returns a snapshot of the issue's current state, not a discrete action, so # event.type is inferred from timestamps. `creation` applies only while the issue has # never been modified; reopened and closed issues have updated_at > created_at. - append: + tag: append_event_type_5b8105e1 field: event.type value: creation if: ctx.github?.issues?.created_at == ctx.github?.issues?.updated_at # A closed issue or a merged pull request has reached the end of its lifecycle. - append: + tag: append_event_type_70c57ee5 field: event.type value: deletion if: ctx.github?.issues?.closed_at != null || ctx.github?.issues?.pull_request?.merged_at != null # Anything still open but modified since it was created has been edited or reopened. - append: + tag: append_event_type_3417105c field: event.type value: change if: ctx.github?.issues?.created_at != ctx.github?.issues?.updated_at && ctx.github?.issues?.closed_at == null && ctx.github?.issues?.pull_request?.merged_at == null @@ -64,6 +71,7 @@ processors: tag: set_event_reason_from_state_reason ignore_empty_value: true - date: + tag: date_github_issues_updated_at_to_github_issues_updated_at_093ebffd field: github.issues.updated_at formats: - ISO8601 @@ -71,6 +79,7 @@ processors: target_field: github.issues.updated_at if: ctx.github.issues.updated_at != null - date: + tag: date_github_issues_created_at_to_github_issues_created_at_1a606418 field: github.issues.created_at formats: - ISO8601 @@ -78,47 +87,58 @@ processors: target_field: github.issues.created_at if: ctx.github.issues.created_at != null - set: + tag: set_timestamp_783c7f2d field: '@timestamp' copy_from: github.issues.updated_at ignore_empty_value: true - set: + tag: set_event_created_e9a0636f field: event.created copy_from: "@timestamp" ignore_empty_value: true - set: + tag: set_github_issues_is_pr_29e7aae7 field: github.issues.is_pr value: false - set: + tag: set_github_issues_is_pr_8baf3bdf if: ctx.github.issues.pull_request != null field: github.issues.is_pr value: true - dissect: + tag: dissect_github_issues_url_843c793f field: github.issues.url pattern: "https://api.github.com/repos/%{_temp_.owner}/%{_temp_.repository}/issues/%{_temp_.number}" ignore_missing: true - set: + tag: set_github_repository_name_3fd19038 field: github.repository.name value: "{{{_temp_.repository}}}" if: ctx._temp_?.repository != null - set: + tag: set_github_repository_html_url_e09a8fc8 field: github.repository.html_url value: "https://github.com/{{{_temp_.owner}}}/{{{_temp_.repository}}}" if: ctx._temp_?.owner != null && ctx._temp_?.repository != null - set: + tag: set_github_repository_url_cc7c15f2 field: github.repository.url value: "https://api.github.com/repos/{{{_temp_.owner}}}/{{{_temp_.repository}}}" if: ctx._temp_?.owner != null && ctx._temp_?.repository != null - set: + tag: set_github_repository_owner_login_3283116b field: github.repository.owner.login value: "{{{_temp_.owner}}}" if: ctx._temp_?.owner != null - foreach: + tag: foreach_github_issues_assignees_6ee08aae field: github.issues.assignees ignore_missing: true processor: remove: field: ["_ingest._value.node_id", "_ingest._value.avatar_url", "_ingest._value.gravatar_id", "_ingest._value.followers_url", "_ingest._value.following_url", "_ingest._value.gists_url", "_ingest._value.starred_url", "_ingest._value.subscriptions_url", "_ingest._value.organizations_url", "_ingest._value.repos_url", "_ingest._value.events_url", "_ingest._value.received_events_url"] - fingerprint: + tag: fingerprint_aba2b3a0 fields: - github.repository.owner.login - github.repository.name @@ -129,6 +149,7 @@ processors: target_field: "_id" ignore_missing: true - remove: + tag: remove_b2d8d5b5 field: - github.issues.user.node_id - github.issues.user.avatar_url @@ -170,10 +191,12 @@ processors: - github.issues.reactions ignore_missing: true - rename: + tag: rename_github_issues_labels_to__temp__labels_eb146963 target_field: _temp_.labels field: github.issues.labels ignore_missing: true - script: + tag: script_261fc403 lang: painless description: Remove unwanted fields from labels if: ctx._temp_?.labels != null @@ -190,6 +213,7 @@ processors: } ctx.github.issues.labels = labels; - script: + tag: script_bfa45937 lang: painless description: Calculate time taken to close an issue if: ctx.github.issues.closed_at != null @@ -207,15 +231,18 @@ processors: # User ECS fields # ################### - set: + tag: set_user_name_4ce1cced field: user.name copy_from: github.issues.user.login if: ctx.github.issues.user?.login != null - convert: + tag: convert_github_issues_user_id_to_user_id_0895ac4d field: github.issues.user.id type: string target_field: user.id ignore_missing: true - append: + tag: append_user_roles_ba25e906 field: user.roles value: ['site_admin'] if: ctx.github.issues.user?.site_admin != null && ctx.github.issues.user?.site_admin == true @@ -224,11 +251,13 @@ processors: # Related ECS fields # ################### - append: + tag: append_related_user_837e080f field: related.user value: "{{{user.name}}}" if: ctx.user?.name != null allow_duplicates: false - foreach: + tag: foreach_github_issues_assignees_20d89808 field: github.issues.assignees if: ctx.github.issues.assignees != null processor: @@ -240,11 +269,13 @@ processors: # Cleanup # ########### - remove: + tag: remove_96fa0639 field: - _temp_ - github.issues.repository ignore_missing: true - script: + tag: script_4d6993cb lang: painless description: This script processor iterates over the whole document to remove fields with null values. source: | diff --git a/packages/github/data_stream/secret_scanning/elasticsearch/ingest_pipeline/default.yml b/packages/github/data_stream/secret_scanning/elasticsearch/ingest_pipeline/default.yml index d53a46ed43d..e93c06f05f3 100644 --- a/packages/github/data_stream/secret_scanning/elasticsearch/ingest_pipeline/default.yml +++ b/packages/github/data_stream/secret_scanning/elasticsearch/ingest_pipeline/default.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing GitHub audit logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - rename: @@ -18,23 +19,29 @@ processors: description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null - json: + tag: json_event_original_to_github_secret_scanning_a20274e1 field: event.original target_field: github.secret_scanning - fail: + tag: fail_79b9a090 if: "!(ctx.github.secret_scanning instanceof Map)" message: Missing JSON object - set: + tag: set_event_kind_39295792 field: event.kind value: alert - append: + tag: append_event_type_6c34b6ef field: event.type value: creation if: ctx.github?.secret_scanning?.resolved_at == null - append: + tag: append_event_type_edf0ab96 field: event.type value: deletion if: ctx.github?.secret_scanning?.resolved_at != null - fingerprint: + tag: fingerprint_1a586380 fields: - github.secret_scanning.number - github.secret_scanning.updated_at @@ -42,10 +49,12 @@ processors: target_field: "_id" ignore_missing: true - set: + tag: set_event_created_7d3e3ade copy_from: github.secret_scanning.created_at field: event.created if: ctx.github.secret_scanning.created_at != null - date: + tag: date_github_secret_scanning_created_at_to_timestamp_1f72e0a4 field: github.secret_scanning.created_at formats: - ISO8601 @@ -53,6 +62,7 @@ processors: target_field: "@timestamp" if: ctx.github.secret_scanning.created_at != null && ctx.github.secret_scanning.updated_at == null && ctx.github.secret_scanning.resolved_at == null - date: + tag: date_github_secret_scanning_updated_at_to_timestamp_6213f1bc field: github.secret_scanning.updated_at formats: - ISO8601 @@ -60,6 +70,7 @@ processors: target_field: "@timestamp" if: ctx.github.secret_scanning.updated_at != null && ctx.github.secret_scanning.resolved_at == null - date: + tag: date_github_secret_scanning_resolved_at_to_timestamp_32015ab3 field: github.secret_scanning.resolved_at formats: - ISO8601 @@ -67,176 +78,219 @@ processors: target_field: "@timestamp" if: ctx.github.secret_scanning.resolved_at != null - rename: + tag: rename_github_secret_scanning_repository_to__temp_repository_f94e7b96 target_field: _temp.repository field: github.secret_scanning.repository ignore_missing: true - rename: + tag: rename__temp_repository_id_to_github_repository_id_629a4bda field: _temp.repository.id target_field: github.repository.id ignore_missing: true - rename: + tag: rename__temp_repository_name_to_github_repository_name_aa4bc352 field: _temp.repository.name target_field: github.repository.name ignore_missing: true - rename: + tag: rename__temp_repository_full_name_to_github_repository_full_name_0eb51a94 field: _temp.repository.full_name target_field: github.repository.full_name ignore_missing: true - rename: + tag: rename__temp_repository_private_to_github_repository_private_60e9cd10 field: _temp.repository.private target_field: github.repository.private ignore_missing: true - rename: + tag: rename__temp_repository_html_url_to_github_repository_html_url_e875fa96 field: _temp.repository.html_url target_field: github.repository.html_url ignore_missing: true - rename: + tag: rename__temp_repository_url_to_github_repository_url_c4b2aae4 field: _temp.repository.url target_field: github.repository.url ignore_missing: true - rename: + tag: rename__temp_repository_description_to_github_repository_description_8723e3be field: _temp.repository.description target_field: github.repository.description ignore_missing: true - rename: + tag: rename__temp_repository_fork_to_github_repository_fork_67150e06 field: _temp.repository.fork target_field: github.repository.fork ignore_missing: true - rename: + tag: rename__temp_repository_owner_login_to_github_repository_owner_login_34d0c8f2 field: _temp.repository.owner.login target_field: github.repository.owner.login ignore_missing: true - rename: + tag: rename__temp_repository_owner_id_to_github_repository_owner_id_b47cec1e field: _temp.repository.owner.id target_field: github.repository.owner.id ignore_missing: true - rename: + tag: rename__temp_repository_owner_url_to_github_repository_owner_url_e1dc208a field: _temp.repository.owner.url target_field: github.repository.owner.url ignore_missing: true - rename: + tag: rename__temp_repository_owner_html_url_to_github_repository_owner_html_url_0a189a6a field: _temp.repository.owner.html_url target_field: github.repository.owner.html_url ignore_missing: true - rename: + tag: rename__temp_repository_owner_type_to_github_repository_owner_type_1f6b99fa field: _temp.repository.owner.type target_field: github.repository.owner.type ignore_missing: true - rename: + tag: rename__temp_repository_owner_site_admin_to_github_repository_owner_site_admin_101e642e field: _temp.repository.owner.site_admin target_field: github.repository.owner.site_admin ignore_missing: true - rename: + tag: rename_github_secret_scanning_resolved_by_to__temp_resolved_by_0aff0496 target_field: _temp.resolved_by field: github.secret_scanning.resolved_by ignore_missing: true - rename: + tag: rename__temp_resolved_by_name_to_github_secret_scanning_resolved_by_name_58362464 field: _temp.resolved_by.name target_field: github.secret_scanning.resolved_by.name ignore_missing: true - rename: + tag: rename__temp_resolved_by_email_to_github_secret_scanning_resolved_by_email_78c81d86 field: _temp.resolved_by.email target_field: github.secret_scanning.resolved_by.email ignore_missing: true - rename: + tag: rename__temp_resolved_by_login_to_github_secret_scanning_resolved_by_login_b7e91d5e field: _temp.resolved_by.login target_field: github.secret_scanning.resolved_by.login ignore_missing: true - rename: + tag: rename__temp_resolved_by_id_to_github_secret_scanning_resolved_by_id_04123504 field: _temp.resolved_by.id target_field: github.secret_scanning.resolved_by.id ignore_missing: true - rename: + tag: rename__temp_resolved_by_node_id_to_github_secret_scanning_resolved_by_node_id_5a6a502a field: _temp.resolved_by.node_id target_field: github.secret_scanning.resolved_by.node_id ignore_missing: true - rename: + tag: rename__temp_resolved_by_url_to_github_secret_scanning_resolved_by_url_d1f226f6 field: _temp.resolved_by.url target_field: github.secret_scanning.resolved_by.url ignore_missing: true - rename: + tag: rename__temp_resolved_by_html_url_to_github_secret_scanning_resolved_by_html_url_540ceef4 field: _temp.resolved_by.html_url target_field: github.secret_scanning.resolved_by.html_url ignore_missing: true - rename: + tag: rename__temp_resolved_by_type_to_github_secret_scanning_resolved_by_type_f1518322 field: _temp.resolved_by.type target_field: github.secret_scanning.resolved_by.type ignore_missing: true - rename: + tag: rename__temp_resolved_by_site_admin_to_github_secret_scanning_resolved_by_site_admin_3b3ac740 field: _temp.resolved_by.site_admin target_field: github.secret_scanning.resolved_by.site_admin ignore_missing: true - dissect: + tag: dissect_github_secret_scanning_url_5cf2126e field: github.secret_scanning.url pattern: "https://api.github.com/repos/%{_temp.owner}/%{_temp.repository}/secret-scanning/alerts/%{_temp.number}" ignore_missing: true - set: + tag: set_github_repository_html_url_ec5e1981 field: github.repository.html_url value: "https://github.com/{{{_temp.owner}}}/{{{_temp.repository}}}" if: "ctx.github.repository?.html_url == null && ctx._temp?.owner != null && ctx._temp?.repository != null" - set: + tag: set_github_repository_url_f0b322a3 field: github.repository.url value: "https://api.github.com/repos/{{{_temp.owner}}}/{{{_temp.repository}}}" if: "ctx.github.repository?.url == null && ctx._temp?.owner != null && ctx._temp?.repository != null" - rename: + tag: rename__temp_repository_to_github_repository_name_c2e74f39 field: _temp.repository target_field: github.repository.name if: ctx.github.repository?.name == null ignore_missing: true - rename: + tag: rename__temp_owner_to_github_repository_owner_login_f9ff210d field: _temp.owner target_field: github.repository.owner.login if: ctx.github.repository?.owner?.login == null ignore_missing: true - rename: + tag: rename__temp_number_to_github_secret_scanning_number_39346671 field: _temp.number target_field: github.secret_scanning.number if: ctx.github.secret_scanning.number == null ignore_missing: true - lowercase: + tag: lowercase_github_secret_scanning_state_9adb8e01 field: github.secret_scanning.state ignore_missing: true - rename: + tag: rename_github_secret_scanning_push_protection_bypassed_by_to__temp_push_protection_bypassed_by_22c4274e target_field: _temp.push_protection_bypassed_by field: github.secret_scanning.push_protection_bypassed_by ignore_missing: true - rename: + tag: rename__temp_push_protection_bypassed_by_name_to_github_secret_scanning_push_protection_bypassed_by_name_87e0dffc field: _temp.push_protection_bypassed_by.name target_field: github.secret_scanning.push_protection_bypassed_by.name ignore_missing: true - rename: + tag: rename__temp_push_protection_bypassed_by_email_to_github_secret_scanning_push_protection_bypassed_by_email_c936ae6e field: _temp.push_protection_bypassed_by.email target_field: github.secret_scanning.push_protection_bypassed_by.email ignore_missing: true - rename: + tag: rename__temp_push_protection_bypassed_by_login_to_github_secret_scanning_push_protection_bypassed_by_login_6dab27be field: _temp.push_protection_bypassed_by.login target_field: github.secret_scanning.push_protection_bypassed_by.login ignore_missing: true - rename: + tag: rename__temp_push_protection_bypassed_by_id_to_github_secret_scanning_push_protection_bypassed_by_id_a022b15c field: _temp.push_protection_bypassed_by.id target_field: github.secret_scanning.push_protection_bypassed_by.id ignore_missing: true - rename: + tag: rename__temp_push_protection_bypassed_by_node_id_to_github_secret_scanning_push_protection_bypassed_by_node_id_90665176 field: _temp.push_protection_bypassed_by.node_id target_field: github.secret_scanning.push_protection_bypassed_by.node_id ignore_missing: true - rename: + tag: rename__temp_push_protection_bypassed_by_url_to_github_secret_scanning_push_protection_bypassed_by_url_4f762c8e field: _temp.push_protection_bypassed_by.url target_field: github.secret_scanning.push_protection_bypassed_by.url ignore_missing: true - rename: + tag: rename__temp_push_protection_bypassed_by_html_url_to_github_secret_scanning_push_protection_bypassed_by_html_url_910114c8 field: _temp.push_protection_bypassed_by.html_url target_field: github.secret_scanning.push_protection_bypassed_by.html_url ignore_missing: true - rename: + tag: rename__temp_push_protection_bypassed_by_type_to_github_secret_scanning_push_protection_bypassed_by_type_2319312e field: _temp.push_protection_bypassed_by.type target_field: github.secret_scanning.push_protection_bypassed_by.type ignore_missing: true - rename: + tag: rename__temp_push_protection_bypassed_by_site_admin_to_github_secret_scanning_push_protection_bypassed_by_site_admin_45393240 field: _temp.push_protection_bypassed_by.site_admin target_field: github.secret_scanning.push_protection_bypassed_by.site_admin ignore_missing: true - script: + tag: script_68516911 lang: painless description: Mask the secret if hide_secret is true if: ctx.tags != null && ctx.tags.contains('hide_secret') && ctx.github.secret_scanning.secret != null @@ -249,6 +303,7 @@ processors: masked_secret = secret.substring(0,num_start) + masked_value + secret.substring(secret.length()-num_end); ctx.github.secret_scanning.secret = masked_secret; - script: + tag: script_5d972797 lang: painless description: Calculate time taken to resolve the secrets if: ctx.github.secret_scanning.resolved_at != null @@ -266,10 +321,12 @@ processors: # Cleanup # ########### - remove: + tag: remove_1f8c5875 field: - _temp ignore_missing: true - script: + tag: script_4d6993cb lang: painless description: This script processor iterates over the whole document to remove fields with null values. source: | diff --git a/packages/github/data_stream/security_advisories/elasticsearch/ingest_pipeline/default.yml b/packages/github/data_stream/security_advisories/elasticsearch/ingest_pipeline/default.yml index e15654ca3df..1e9da83e327 100644 --- a/packages/github/data_stream/security_advisories/elasticsearch/ingest_pipeline/default.yml +++ b/packages/github/data_stream/security_advisories/elasticsearch/ingest_pipeline/default.yml @@ -48,6 +48,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: >- Processor {{{_ingest.on_failure_processor_type}}} with tag @@ -75,6 +76,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_de29c875 field: error.message value: >- Processor {{{_ingest.on_failure_processor_type}}} with tag @@ -98,6 +100,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2a7f5299 field: error.message value: >- Processor {{{_ingest.on_failure_processor_type}}} with tag @@ -116,6 +119,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_1218d582 field: error.message value: >- Processor {{{_ingest.on_failure_processor_type}}} with tag @@ -147,6 +151,7 @@ processors: type: float on_failure: - append: + tag: append_error_message_ddddf548 field: error.message value: >- Processor {{{_ingest.on_failure_processor_type}}} with tag @@ -161,6 +166,7 @@ processors: type: float on_failure: - append: + tag: append_error_message_c650efae field: error.message value: >- Processor {{{_ingest.on_failure_processor_type}}} with tag @@ -184,14 +190,17 @@ processors: copy_from: github.security_advisory.json.html_url ignore_empty_value: true - uri_parts: + tag: uri_parts_json_html_url_b5189973 field: json.html_url if: ctx.json?.html_url != null && ctx.json.html_url != '' on_failure: - set: + tag: set_url_original_296a54e4 field: url.original value: '{{{json.html_url}}}' if: ctx.json?.html_url != null && ctx.json.html_url != '' - set: + tag: set_url_full_8a0b2565 field: url.full value: '{{{url.original}}}' if: ctx.url?.original != null && ctx.url.original != '' @@ -280,7 +289,7 @@ processors: tag: remove_json field: - json - + # Cleanup - script: tag: script_to_drop_null_values @@ -301,10 +310,12 @@ processors: } drop(ctx); - set: + tag: set_event_kind_92954dfa field: event.kind value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/github/manifest.yml b/packages/github/manifest.yml index cac4c5e511c..1347e889211 100644 --- a/packages/github/manifest.yml +++ b/packages/github/manifest.yml @@ -1,6 +1,6 @@ name: github title: GitHub -version: "2.25.2" +version: "2.25.3" description: Collect logs from GitHub with Elastic Agent. type: integration format_version: "3.4.0" diff --git a/packages/gitlab/changelog.yml b/packages/gitlab/changelog.yml index 6efa7931d7f..6774a9b3ecd 100644 --- a/packages/gitlab/changelog.yml +++ b/packages/gitlab/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.6.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "2.6.1" changes: - description: Normalize `custom_message` field to handle both string and object values. diff --git a/packages/gitlab/data_stream/api/elasticsearch/ingest_pipeline/default.yml b/packages/gitlab/data_stream/api/elasticsearch/ingest_pipeline/default.yml index 28c383d7585..8018f49cc9d 100644 --- a/packages/gitlab/data_stream/api/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gitlab/data_stream/api/elasticsearch/ingest_pipeline/default.yml @@ -6,6 +6,7 @@ processors: tag: set_ecs_version value: 8.11.0 - set: + tag: set_event_kind_de80643c field: event.kind value: event - rename: @@ -19,7 +20,8 @@ processors: ignore_missing: true tag: remove_message if: ctx.event?.original != null -- drop: +- drop: + tag: drop_3b0973c1 if: ctx.event.original.startsWith('#') description: Drop if logline contains header(s), which startswith `#`. - json: @@ -34,13 +36,16 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_82935f46 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_gitlab_api_severity_to_log_level_0542d7c2 field: gitlab.api.severity target_field: log.level ignore_missing: true - script: + tag: script_06fa2bd8 lang: painless source: |- if (ctx.gitlab?.api?.duration_s != null) { @@ -54,9 +59,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_db_1f19a206 field: gitlab.api.db ignore_missing: true - append: + tag: append_error_message_c8a0b0d4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -66,9 +73,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_view_3850bf48 field: gitlab.api.view ignore_missing: true - append: + tag: append_error_message_8923b66f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -85,28 +94,35 @@ processors: } ctx.gitlab.api.params = keyValuePairs; - rename: + tag: rename_gitlab_api_status_to_http_response_status_code_4d1ffe5e field: gitlab.api.status target_field: http.response.status_code ignore_missing: true - rename: + tag: rename_gitlab_api_method_to_http_request_method_17b56cec field: gitlab.api.method target_field: http.request.method ignore_missing: true - rename: + tag: rename_gitlab_api_path_to_url_path_ce1a2d7a field: gitlab.api.path target_field: url.path ignore_missing: true - rename: + tag: rename_gitlab_api_host_to_url_domain_c0323fb8 field: gitlab.api.host target_field: url.domain ignore_missing: true - dot_expander: + tag: dot_expander_meta_caller_id_db93dc40 path: gitlab.api field: meta.caller_id - dot_expander: + tag: dot_expander_meta_client_id_e3d08f84 path: gitlab.api field: meta.client_id - dot_expander: + tag: dot_expander_meta_feature_category_fa66bc4c path: gitlab.api field: meta.feature_category - dot_expander: @@ -118,26 +134,33 @@ processors: tag: dot_expander_meta_gl_organization_id field: meta.organization_id - dot_expander: + tag: dot_expander_meta_remote_ip_3bdde189 path: gitlab.api field: meta.remote_ip - convert: + tag: convert_gitlab_api_meta_remote_ip_1d185ef7 field: gitlab.api.meta.remote_ip type: ip ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_meta_remote_ip_2d91d7f2 field: gitlab.api.meta.remote_ip ignore_missing: true - append: + tag: append_error_message_18efb825 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - dot_expander: + tag: dot_expander_meta_user_241f7508 path: gitlab.api field: meta.user - dot_expander: + tag: dot_expander_meta_user_id_3bb65088 path: gitlab.api field: meta.user_id - rename: + tag: rename_gitlab_api_meta_caller_id_to_event_provider_4abbd092 field: gitlab.api.meta.caller_id target_field: event.provider ignore_missing: true @@ -148,6 +171,7 @@ processors: target_field: source.ip separator: ', ?' - foreach: + tag: foreach_source_ip_bc825b65 field: source.ip if: ctx.source?.ip instanceof List ignore_failure: true @@ -161,6 +185,7 @@ processors: field: _ingest._value ignore_missing: true - remove: + tag: remove_gitlab_api_remote_ip_a48672a5 field: gitlab.api.remote_ip ignore_missing: true - foreach: @@ -175,22 +200,27 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - rename: + tag: rename_gitlab_api_ua_to_user_agent_original_5b852e67 field: gitlab.api.ua target_field: user_agent.original ignore_missing: true - rename: + tag: rename_gitlab_api_pid_to_process_pid_446230b8 field: gitlab.api.pid target_field: process.pid ignore_missing: true - rename: + tag: rename_gitlab_api_user_id_to_user_id_70db25f8 field: gitlab.api.user_id target_field: user.id ignore_missing: true - convert: + tag: convert_user_id_c132c028 field: user.id type: string ignore_missing: true - rename: + tag: rename_gitlab_api_username_to_user_name_52885247 field: gitlab.api.username target_field: user.name ignore_missing: true @@ -219,32 +249,38 @@ processors: target_field: client.as.organization.name ignore_missing: true - append: + tag: append_event_category_d097611d field: event.category value: - api allow_duplicates: false - append: + tag: append_event_type_6adea021 field: event.type value: - info allow_duplicates: false - append: + tag: append_event_category_2c6f4651 field: event.category value: - database allow_duplicates: false if: ctx.gitlab?.api?.db_count != null && ctx.gitlab?.api?.db_count > 0 - append: + tag: append_event_type_058d222b field: event.type value: - error allow_duplicates: false if: ctx.http?.response?.status_code != null && (ctx.http.response.status_code < 200 || ctx.http.response.status_code >= 400) - set: + tag: set_event_kind_92954dfa field: event.kind value: pipeline_error if: ctx.error?.message != null - script: + tag: script_a3eb5add description: Drops null/empty values recursively. lang: painless source: | @@ -268,9 +304,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_content_length_dc485fe5 field: gitlab.api.content_length ignore_missing: true - append: + tag: append_error_message_484e6e0f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -280,9 +318,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_db_ci_replica_txn_max_duration_s_0b5365e1 field: gitlab.api.db_ci_replica_txn_max_duration_s ignore_missing: true - append: + tag: append_error_message_e5393d22 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -292,9 +332,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_db_ci_replica_write_count_a905d053 field: gitlab.api.db_ci_replica_write_count ignore_missing: true - append: + tag: append_error_message_ca6a601b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -304,9 +346,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_db_ci_txn_max_duration_s_03f35a0d field: gitlab.api.db_ci_txn_max_duration_s ignore_missing: true - append: + tag: append_error_message_37aa0d7f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -316,9 +360,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_db_ci_write_count_1d694707 field: gitlab.api.db_ci_write_count ignore_missing: true - append: + tag: append_error_message_d9995b82 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -328,9 +374,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_db_main_replica_txn_max_duration_s_210887f1 field: gitlab.api.db_main_replica_txn_max_duration_s ignore_missing: true - append: + tag: append_error_message_71b0b465 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -340,9 +388,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_db_main_replica_write_count_d7254383 field: gitlab.api.db_main_replica_write_count ignore_missing: true - append: + tag: append_error_message_faf78758 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -352,9 +402,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_db_main_txn_max_duration_s_dd76c78d field: gitlab.api.db_main_txn_max_duration_s ignore_missing: true - append: + tag: append_error_message_36dd7840 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -364,9 +416,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_db_main_write_count_80163c97 field: gitlab.api.db_main_write_count ignore_missing: true - append: + tag: append_error_message_18264b21 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -376,9 +430,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_db_primary_txn_max_duration_s_435e72c7 field: gitlab.api.db_primary_txn_max_duration_s ignore_missing: true - append: + tag: append_error_message_b5a9c0b9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -388,9 +444,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_db_primary_write_count_7ae62e91 field: gitlab.api.db_primary_write_count ignore_missing: true - append: + tag: append_error_message_dfeed09e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -400,9 +458,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_db_replica_txn_max_duration_s_20856957 field: gitlab.api.db_replica_txn_max_duration_s ignore_missing: true - append: + tag: append_error_message_d8489bbf field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -412,9 +472,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_db_replica_write_count_74e8e9b1 field: gitlab.api.db_replica_write_count ignore_missing: true - append: + tag: append_error_message_bb6ad5c4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -424,9 +486,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_exclusive_lock_hold_duration_s_1445bfa9 field: gitlab.api.exclusive_lock_hold_duration_s ignore_missing: true - append: + tag: append_error_message_f75ea5fc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -436,9 +500,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_exclusive_lock_requested_count_06553521 field: gitlab.api.exclusive_lock_requested_count ignore_missing: true - append: + tag: append_error_message_c1df2aa4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -448,9 +514,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_exclusive_lock_wait_duration_s_bf057995 field: gitlab.api.exclusive_lock_wait_duration_s ignore_missing: true - append: + tag: append_error_message_8310f564 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -460,9 +528,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_gitaly_duration_s_abd03513 field: gitlab.api.gitaly_duration_s ignore_missing: true - append: + tag: append_error_message_2425f3be field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -504,9 +574,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_queues_calls_06aad76d field: gitlab.api.redis_queues_calls ignore_missing: true - append: + tag: append_error_message_3fac5963 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -516,9 +588,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_queues_duration_s_77080e17 field: gitlab.api.redis_queues_duration_s ignore_missing: true - append: + tag: append_error_message_366287aa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -528,9 +602,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_queues_metadata_calls_8396973b field: gitlab.api.redis_queues_metadata_calls ignore_missing: true - append: + tag: append_error_message_c588bc99 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -540,9 +616,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_queues_metadata_duration_s_f77bf659 field: gitlab.api.redis_queues_metadata_duration_s ignore_missing: true - append: + tag: append_error_message_cf0f60ee field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -552,9 +630,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_path_traversal_check_duration_s_a51d5f27 field: gitlab.api.path_traversal_check_duration_s ignore_missing: true - append: + tag: append_error_message_6b592b0b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -564,9 +644,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_queues_metadata_read_bytes_1d069be1 field: gitlab.api.redis_queues_metadata_read_bytes ignore_missing: true - append: + tag: append_error_message_62321e30 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -576,9 +658,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_queues_metadata_write_bytes_a937273b field: gitlab.api.redis_queues_metadata_write_bytes ignore_missing: true - append: + tag: append_error_message_75d40937 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -588,9 +672,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_queues_read_bytes_17bca683 field: gitlab.api.redis_queues_read_bytes ignore_missing: true - append: + tag: append_error_message_4c3ea1d0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -600,9 +686,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_queues_write_bytes_8d389e19 field: gitlab.api.redis_queues_write_bytes ignore_missing: true - append: + tag: append_error_message_0497148d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -612,9 +700,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_rate_limiting_calls_4133314f field: gitlab.api.redis_rate_limiting_calls ignore_missing: true - append: + tag: append_error_message_78c1a5df field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -624,9 +714,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_rate_limiting_duration_s_1b5214dd field: gitlab.api.redis_rate_limiting_duration_s ignore_missing: true - append: + tag: append_error_message_e392e318 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -636,9 +728,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_rate_limiting_read_bytes_46b925ad field: gitlab.api.redis_rate_limiting_read_bytes ignore_missing: true - append: + tag: append_error_message_0ac87572 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -648,9 +742,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_rate_limiting_write_bytes_eac38bdf field: gitlab.api.redis_rate_limiting_write_bytes ignore_missing: true - append: + tag: append_error_message_e81065fd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -660,9 +756,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_repository_cache_allowed_cross_slot_calls_8efe488b field: gitlab.api.redis_repository_cache_allowed_cross_slot_calls ignore_missing: true - append: + tag: append_error_message_1c530953 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -672,9 +770,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_repository_cache_calls_b952d64d field: gitlab.api.redis_repository_cache_calls ignore_missing: true - append: + tag: append_error_message_058284f6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -684,9 +784,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_repository_cache_duration_s_f5ee745f field: gitlab.api.redis_repository_cache_duration_s ignore_missing: true - append: + tag: append_error_message_479bc067 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -696,9 +798,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_repository_cache_read_bytes_8b86e683 field: gitlab.api.redis_repository_cache_read_bytes ignore_missing: true - append: + tag: append_error_message_514f4251 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -708,9 +812,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_repository_cache_write_bytes_ace93ad9 field: gitlab.api.redis_repository_cache_write_bytes ignore_missing: true - append: + tag: append_error_message_f84ab664 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -720,9 +826,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_shared_state_calls_b7a37c81 field: gitlab.api.redis_shared_state_calls ignore_missing: true - append: + tag: append_error_message_aa989fcc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -732,9 +840,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_shared_state_duration_s_ee8a227b field: gitlab.api.redis_shared_state_duration_s ignore_missing: true - append: + tag: append_error_message_bcb82f01 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -744,9 +854,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_shared_state_read_bytes_4546658f field: gitlab.api.redis_shared_state_read_bytes ignore_missing: true - append: + tag: append_error_message_e98d007b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -756,9 +868,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_redis_shared_state_write_bytes_4222bb95 field: gitlab.api.redis_shared_state_write_bytes ignore_missing: true - append: + tag: append_error_message_27b8c6a2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -784,8 +898,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} - in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - append: field: tags value: preserve_original_event diff --git a/packages/gitlab/data_stream/application/elasticsearch/ingest_pipeline/default.yml b/packages/gitlab/data_stream/application/elasticsearch/ingest_pipeline/default.yml index c07c1b98934..5996392ec20 100644 --- a/packages/gitlab/data_stream/application/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gitlab/data_stream/application/elasticsearch/ingest_pipeline/default.yml @@ -2,9 +2,11 @@ description: Pipeline for processing application logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: 8.11.0 - set: + tag: set_event_kind_de80643c field: event.kind value: event - rename: @@ -18,7 +20,8 @@ processors: ignore_missing: true tag: remove_message if: ctx.event?.original != null - - drop: + - drop: + tag: drop_3b0973c1 if: ctx.event.original.startsWith('#') description: Drop if logline contains header(s), which startswith `#`. - json: @@ -34,10 +37,12 @@ processors: tag: date_event_created_time_epoch on_failure: - append: + tag: append_error_message_3d2919b3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' if: ctx.gitlab?.application?.time != null - remove: + tag: remove_gitlab_application_time_6a62da82 field: gitlab.application.time ignore_missing: true - set: @@ -71,13 +76,16 @@ processors: if: ctx.gitlab?.application?.severity == 'UNKNOWN' tag: set_severity_5 - remove: + tag: remove_gitlab_application_severity_45806750 field: gitlab.application.severity ignore_missing: true - rename: + tag: rename_gitlab_application_correlation_id_to_event_id_ef19d222 field: gitlab.application.correlation_id target_field: event.id ignore_missing: true - rename: + tag: rename_gitlab_application_pid_to_process_pid_7bf152a6 field: gitlab.application.pid target_field: process.pid ignore_missing: true @@ -101,6 +109,7 @@ processors: } } - dot_expander: + tag: dot_expander_*_57c81ca6 field: "*" path: gitlab.application - script: @@ -123,16 +132,20 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_application_namespace_id_8505a1fa field: gitlab.application.namespace_id ignore_missing: true - append: + tag: append_error_message_1689da49 field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_gitlab_application_meta_user_to_user_name_e32fe3a7 field: gitlab.application.meta.user target_field: user.name ignore_missing: true - convert: + tag: convert_gitlab_application_meta_user_id_02a37dc9 field: gitlab.application.meta.user_id type: string ignore_missing: true @@ -143,30 +156,37 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_application_user_id_b9ad7fa2 field: gitlab.application.user_id ignore_missing: true - append: + tag: append_error_message_98b04b27 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_gitlab_application_meta_user_id_to_user_id_4e8782c5 field: gitlab.application.meta.user_id target_field: user.id ignore_missing: true - append: + tag: append_related_user_7f407fef field: related.user value: '{{{user.id}}}' if: ctx.user?.id != null allow_duplicates: false - append: + tag: append_related_user_837e080f field: related.user value: '{{{user.name}}}' if: ctx.user?.name != null allow_duplicates: false - rename: + tag: rename_gitlab_application_mergeability_merge_request_id_to_gitlab_application_mergeability_merge_request_id_a345e139 field: gitlab.application.mergeability_merge_request_id target_field: gitlab.application.mergeability.merge_request_id ignore_missing: true - rename: + tag: rename_gitlab_application_mergeability_project_id_to_gitlab_application_mergeability_project_id_bcaa0d2d field: gitlab.application.mergeability_project_id target_field: gitlab.application.mergeability.project_id ignore_missing: true @@ -177,9 +197,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_application_duration_s_9a177787 field: gitlab.application.duration_s ignore_missing: true - append: + tag: append_error_message_01b61563 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -232,20 +254,25 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_0ca09779 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_gitlab_application_meta_remote_ip_5ce7515f field: gitlab.application.meta.remote_ip type: ip ignore_missing: true on_failure: - remove: + tag: remove_gitlab_application_meta_remote_ip_a722d4d4 field: gitlab.application.meta.remote_ip ignore_missing: true - append: + tag: append_error_message_b2bb4bbd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_gitlab_application_meta_remote_ip_to_client_ip_f714f91e field: gitlab.application.meta.remote_ip target_field: client.ip ignore_missing: true @@ -274,14 +301,17 @@ processors: target_field: client.as.organization.name ignore_missing: true - set: + tag: set_client_address_7c2df43a field: client.address copy_from: client.ip if: ctx.client?.ip != null - set: + tag: set_source_f7fb1ee5 field: source copy_from: client if: ctx.client?.ip != null - append: + tag: append_related_ip_3726c45d field: related.ip value: '{{{client.ip}}}' if: ctx.client?.ip != null @@ -303,18 +333,22 @@ processors: tag: 'append_iam_into_event_category' if: ctx.gitlab?.application?.message != null && ctx.gitlab.application.message.contains('Group') - set: + tag: set_event_outcome_75402cb0 field: event.outcome value: failure if: ctx.gitlab?.application?.message != null && ctx.gitlab.application.message.contains("Failed Login:") - set: + tag: set_event_outcome_2f7cca02 field: event.outcome value: success if: ctx.gitlab?.application?.message != null && ctx.gitlab.application.message.contains("Successful Login:") - append: + tag: append_event_type_eb764948 field: event.type value: creation if: 'ctx.gitlab?.application?.message != null && (ctx.gitlab.application.message.contains("created a new project") || ctx.gitlab.application.message.contains(" was created"))' - append: + tag: append_event_type_a048f3f4 field: event.type value: deletion if: ctx.gitlab?.application?.message != null && ctx.gitlab.application.message.contains(" was deleted") @@ -323,10 +357,12 @@ processors: field: event.type value: info - append: + tag: append_event_category_8aaf2294 field: event.category value: configuration if: ' ctx.gitlab?.application?.message != null && !(ctx.gitlab?.application?.message.contains("Failed Login:") || ctx.gitlab?.application?.message.contains("Successful Login:"))' - script: + tag: script_a3eb5add description: Drops null/empty values recursively. lang: painless source: | @@ -344,10 +380,12 @@ processors: } dropEmptyFields(ctx); - set: + tag: set_event_kind_92954dfa field: event.kind value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false @@ -355,8 +393,8 @@ processors: on_failure: - append: field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} - in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: field: event.kind value: pipeline_error diff --git a/packages/gitlab/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/gitlab/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 2be1bb0d2cc..e000a957230 100644 --- a/packages/gitlab/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gitlab/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -2,9 +2,11 @@ description: Pipeline for processing audit logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: 8.11.0 - set: + tag: set_event_kind_de80643c field: event.kind value: event - rename: @@ -18,7 +20,8 @@ processors: ignore_missing: true tag: remove_message if: ctx.event?.original != null - - drop: + - drop: + tag: drop_3b0973c1 if: ctx.event.original.startsWith('#') description: Drop if logline contains header(s), which startswith `#`. - json: @@ -34,10 +37,12 @@ processors: tag: date_event_created_time_epoch on_failure: - append: + tag: append_error_message_e407d4e7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' if: ctx.gitlab?.audit?.time != null - remove: + tag: remove_gitlab_audit_time_e9f0736f field: gitlab.audit.time ignore_missing: true - set: @@ -71,9 +76,11 @@ processors: if: ctx.gitlab?.audit?.severity == 'UNKNOWN' tag: set_severity_5 - remove: + tag: remove_gitlab_audit_severity_a34e6fe9 field: gitlab.audit.severity ignore_missing: true - rename: + tag: rename_gitlab_audit_correlation_id_to_event_id_b0cdaaa3 field: gitlab.audit.correlation_id target_field: event.id ignore_missing: true @@ -100,20 +107,25 @@ processors: copy_from: gitlab.audit.details.custom_message_object.action ignore_empty_value: true - dot_expander: + tag: dot_expander_*_df8292b5 path: gitlab.audit field: "*" - convert: + tag: convert_gitlab_audit_meta_remote_ip_be914348 field: gitlab.audit.meta.remote_ip type: ip ignore_missing: true on_failure: - remove: + tag: remove_gitlab_audit_meta_remote_ip_461c171a field: gitlab.audit.meta.remote_ip ignore_missing: true - append: + tag: append_error_message_1633f4c3 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_gitlab_audit_meta_remote_ip_to_client_ip_10f614c1 field: gitlab.audit.meta.remote_ip target_field: client.ip ignore_missing: true @@ -142,27 +154,33 @@ processors: target_field: client.as.organization.name ignore_missing: true - set: + tag: set_client_address_7c2df43a field: client.address copy_from: client.ip if: ctx.client?.ip != null - set: + tag: set_source_f7fb1ee5 field: source copy_from: client if: ctx.client?.ip != null - append: + tag: append_related_ip_3726c45d field: related.ip value: '{{{client.ip}}}' if: ctx.client?.ip != null allow_duplicates: false - convert: + tag: convert_gitlab_audit_author_id_bb0cb421 field: gitlab.audit.author_id type: string ignore_missing: true - rename: + tag: rename_gitlab_audit_author_id_to_user_id_e136c633 field: gitlab.audit.author_id target_field: user.id ignore_missing: true - rename: + tag: rename_gitlab_audit_author_name_to_user_name_a80973eb field: gitlab.audit.author_name target_field: user.name ignore_missing: true @@ -172,42 +190,50 @@ processors: - "ISO8601" target_field: "@timestamp" timezone: "UTC" - tag: date_event_created_time_epoch + tag: date_gitlab_audit_created_at_to_timestamp_8a70115b on_failure: - append: + tag: append_error_message_9473d7cd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' if: ctx.gitlab?.audit?.created_at != null - append: + tag: append_related_user_7f407fef field: related.user value: '{{{user.id}}}' if: ctx.user?.id != null allow_duplicates: false - append: + tag: append_related_user_837e080f field: related.user value: '{{{user.name}}}' if: ctx.user?.name != null allow_duplicates: false - append: + tag: append_related_user_6a3b5808 field: related.user value: '{{{gitlab.audit.target_id}}}' if: ctx.gitlab?.audit?.target_id != null && ctx.gitlab.audit.target_type == "User" allow_duplicates: false - append: + tag: append_related_user_da886592 field: related.user value: '{{{gitlab.audit.target_details}}}' if: ctx.gitlab?.audit?.target_details != null && ctx.gitlab.audit.target_type == "User" allow_duplicates: false - append: + tag: append_related_user_038455c2 field: related.user value: '{{{gitlab.audit.entity_id}}}' if: ctx.gitlab?.audit?.entity_id != null && ctx.gitlab.audit.entity_type == "User" allow_duplicates: false - append: + tag: append_event_category_59b4d911 field: event.category value: web allow_duplicates: false - append: + tag: append_event_type_3f43a39d field: event.type value: info allow_duplicates: false @@ -230,11 +256,13 @@ processors: } drop(ctx); - append: + tag: append_event_kind_d2f25e35 field: event.kind value: pipeline_error allow_duplicates: false if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/gitlab/data_stream/auth/elasticsearch/ingest_pipeline/default.yml b/packages/gitlab/data_stream/auth/elasticsearch/ingest_pipeline/default.yml index 05b66896570..3cacfe1c4c3 100644 --- a/packages/gitlab/data_stream/auth/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gitlab/data_stream/auth/elasticsearch/ingest_pipeline/default.yml @@ -2,9 +2,11 @@ description: Pipeline for processing auth logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: 8.11.0 - set: + tag: set_event_kind_de80643c field: event.kind value: event - rename: @@ -18,7 +20,8 @@ processors: ignore_missing: true tag: remove_message if: ctx.event?.original != null - - drop: + - drop: + tag: drop_3b0973c1 if: ctx.event.original.startsWith('#') description: Drop if logline contains header(s), which startswith `#`. - json: @@ -34,10 +37,12 @@ processors: tag: date_event_created_time_epoch on_failure: - append: + tag: append_error_message_9f34d795 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' if: ctx.gitlab?.auth?.time != null - remove: + tag: remove_gitlab_auth_time_2e702b42 field: gitlab.auth.time ignore_missing: true - set: @@ -71,62 +76,77 @@ processors: if: ctx.gitlab?.auth?.severity == 'UNKNOWN' tag: set_severity_5 - remove: + tag: remove_gitlab_auth_severity_796c7e10 field: gitlab.auth.severity ignore_missing: true - rename: + tag: rename_gitlab_auth_correlation_id_to_event_id_6105bba2 field: gitlab.auth.correlation_id target_field: event.id - ignore_missing: true + ignore_missing: true - dot_expander: + tag: dot_expander_meta_user_b6aaeece path: gitlab.auth field: meta.user - convert: + tag: convert_gitlab_auth_user_id_15a26074 field: gitlab.auth.user_id type: string ignore_missing: true - rename: + tag: rename_gitlab_auth_user_id_to_user_id_12314bee field: gitlab.auth.user_id target_field: user.id ignore_missing: true - - rename: + - rename: + tag: rename_gitlab_auth_request_method_to_http_request_method_f485dfec field: gitlab.auth.request_method target_field: http.request.method ignore_missing: true - rename: + tag: rename_gitlab_auth_status_to_http_response_status_code_e95ed6e0 field: gitlab.auth.status target_field: http.response.status_code ignore_missing: true - rename: + tag: rename_gitlab_auth_pid_to_process_pid_b33f8c26 field: gitlab.auth.pid target_field: process.pid ignore_missing: true - set: + tag: set_tmp_url_orig_3acb4090 field: tmp.url_orig value: "https://tmp.gitlab{{{gitlab.auth.path}}}" if: ctx.gitlab?.auth?.path != null - uri_parts: + tag: uri_parts_tmp_url_orig_to_tmp_url_81259b73 field: tmp.url_orig target_field: tmp.url keep_original: false ignore_missing: true - rename: + tag: rename_tmp_url_path_to_url_path_c2d99437 field: tmp.url.path target_field: url.path ignore_missing: true - rename: + tag: rename_tmp_url_query_to_url_query_16a2a373 field: tmp.url.query target_field: url.query ignore_missing: true - rename: + tag: rename_tmp_url_fragment_to_url_frament_6add0f38 field: tmp.url.fragment target_field: url.frament ignore_missing: true - remove: - field: + tag: remove_b92a4353 + field: - tmp - gitlab.auth.path ignore_missing: true - rename: + tag: rename_gitlab_auth_remote_ip_to_client_ip_1a6d1a51 field: gitlab.auth.remote_ip target_field: client.ip ignore_missing: true @@ -155,23 +175,28 @@ processors: target_field: client.as.organization.name ignore_missing: true - set: + tag: set_client_address_7c2df43a field: client.address copy_from: client.ip if: ctx.client?.ip != null - set: + tag: set_source_f7fb1ee5 field: source copy_from: client if: ctx.client?.ip != null - append: + tag: append_related_ip_3726c45d field: related.ip value: '{{{client.ip}}}' if: ctx.client?.ip != null allow_duplicates: false - append: + tag: append_event_category_59b4d911 field: event.category value: web allow_duplicates: false - append: + tag: append_event_type_3f43a39d field: event.type value: info allow_duplicates: false @@ -194,11 +219,13 @@ processors: } drop(ctx); - append: + tag: append_event_kind_d2f25e35 field: event.kind value: pipeline_error allow_duplicates: false if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/gitlab/data_stream/pages/elasticsearch/ingest_pipeline/default.yml b/packages/gitlab/data_stream/pages/elasticsearch/ingest_pipeline/default.yml index 0d7265613b8..ce3dadf4d4a 100644 --- a/packages/gitlab/data_stream/pages/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gitlab/data_stream/pages/elasticsearch/ingest_pipeline/default.yml @@ -16,16 +16,20 @@ processors: ignore_missing: true tag: remove_message if: ctx.event?.original != null - - drop: + - drop: + tag: drop_a8ea228b if: ctx.event?.original?.startsWith('#') description: Drop if logline contains header(s), which startswith `#`. - append: + tag: append_event_category_4595ee28 field: event.category value: web - append: + tag: append_event_type_8a66ccaa field: event.type value: info - set: + tag: set_event_kind_de80643c field: event.kind value: event - json: @@ -41,13 +45,16 @@ processors: tag: date_event_created_time_epoch on_failure: - append: + tag: append_error_message_e1792017 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' if: ctx.gitlab?.pages?.time != null - remove: + tag: remove_gitlab_pages_time_72f76244 field: gitlab.pages.time ignore_missing: true - lowercase: + tag: lowercase_gitlab_pages_level_3f4d571e field: gitlab.pages.level ignore_missing: true - set: @@ -91,29 +98,36 @@ processors: if: ctx.gitlab?.pages?.level == 'debug' tag: set_level_7 - remove: + tag: remove_gitlab_pages_level_10eff751 field: gitlab.pages.level ignore_missing: true - rename: + tag: rename_gitlab_pages_correlation_id_to_event_id_8a3648bc field: gitlab.pages.correlation_id target_field: event.id ignore_missing: true - - rename: + - rename: + tag: rename_gitlab_pages_gid_to_group_id_f707521b field: gitlab.pages.gid target_field: group.id ignore_missing: true - convert: + tag: convert_group_id_5cd2a948 field: group.id type: string ignore_missing: true - - rename: + - rename: + tag: rename_gitlab_pages_in-place_to_gitlab_pages_in_place_0d6e7620 field: gitlab.pages.in-place target_field: gitlab.pages.in_place ignore_missing: true - - rename: + - rename: + tag: rename_gitlab_pages_uid_to_user_id_d21ba517 field: gitlab.pages.uid target_field: user.id ignore_missing: true - - convert: + - convert: + tag: convert_user_id_c132c028 field: user.id type: string ignore_missing: true @@ -129,9 +143,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_pages_duration_ms_9d989e3f field: gitlab.pages.duration_ms ignore_missing: true - append: + tag: append_error_message_1a743fd4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -151,9 +167,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_pages_in_place_a42022cd field: gitlab.pages.in_place ignore_missing: true - append: + tag: append_error_message_045c6b72 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -169,9 +187,11 @@ processors: if: ctx.gitlab?.pages?.listen_addr?.ip != '' on_failure: - remove: + tag: remove_gitlab_pages_listen_addr_ip_d7643e04 field: gitlab.pages.listen_addr.ip ignore_missing: true - append: + tag: append_error_message_217d3616 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -191,9 +211,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_pages_listen_addr_port_809d0f44 field: gitlab.pages.listen_addr.port ignore_missing: true - append: + tag: append_error_message_0197abe4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -213,9 +235,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_pages_pages_https_2d5941ab field: gitlab.pages.pages_https ignore_missing: true - append: + tag: append_error_message_78547f8d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - dissect: @@ -225,6 +249,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f32fe504 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -234,9 +259,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_pages_read_bytes_5510939d field: gitlab.pages.read_bytes ignore_missing: true - append: + tag: append_error_message_d610a729 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -256,6 +283,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4d109d0a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -271,9 +299,11 @@ processors: if: ctx.gitlab?.pages?.remote_ip != '' on_failure: - remove: + tag: remove_gitlab_pages_remote_ip_be681658 field: gitlab.pages.remote_ip ignore_missing: true - append: + tag: append_error_message_919ae0b2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -288,9 +318,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_pages_status_3182df65 field: gitlab.pages.status ignore_missing: true - append: + tag: append_error_message_0acdebbb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -305,9 +337,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_pages_ttfb_ms_9ab64e43 field: gitlab.pages.ttfb_ms ignore_missing: true - append: + tag: append_error_message_14d99ae8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -317,9 +351,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_pages_written_bytes_289535a3 field: gitlab.pages.written_bytes ignore_missing: true - append: + tag: append_error_message_b3661176 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -327,11 +363,13 @@ processors: tag: set_client_bytes_from_pages_written_bytes copy_from: gitlab.pages.written_bytes ignore_empty_value: true - - rename: + - rename: + tag: rename_gitlab_pages_msg_to_message_a7ee9c5f field: gitlab.pages.msg target_field: message ignore_missing: true - append: + tag: append_related_user_7f407fef field: related.user value: '{{{user.id}}}' if: ctx.user?.id != null @@ -355,11 +393,13 @@ processors: } drop(ctx); - append: + tag: append_event_kind_d2f25e35 field: event.kind value: pipeline_error allow_duplicates: false if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/gitlab/data_stream/production/elasticsearch/ingest_pipeline/default.yml b/packages/gitlab/data_stream/production/elasticsearch/ingest_pipeline/default.yml index 371fd33673c..59e61cc6103 100644 --- a/packages/gitlab/data_stream/production/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gitlab/data_stream/production/elasticsearch/ingest_pipeline/default.yml @@ -6,6 +6,7 @@ processors: tag: set_ecs_version value: 8.11.0 - set: + tag: set_event_kind_de80643c field: event.kind value: event - rename: @@ -19,7 +20,8 @@ processors: ignore_missing: true tag: remove_message if: ctx.event?.original != null -- drop: +- drop: + tag: drop_3b0973c1 if: ctx.event.original.startsWith('#') description: Drop if logline contains header(s), which startswith `#`. - json: @@ -27,24 +29,30 @@ processors: tag: json_original target_field: gitlab.production - rename: + tag: rename_gitlab_production_method_to_http_request_method_c3d58fbd field: gitlab.production.method target_field: http.request.method ignore_missing: true - rename: + tag: rename_gitlab_production_path_to_url_path_1f2bb67d field: gitlab.production.path target_field: url.path ignore_missing: true - rename: + tag: rename_gitlab_production_location_to_url_full_c6ab42a3 field: gitlab.production.location target_field: url.full ignore_missing: true - dot_expander: + tag: dot_expander_meta_caller_id_2dbed491 path: gitlab.production field: meta.caller_id - dot_expander: + tag: dot_expander_meta_client_id_733814cd path: gitlab.production field: meta.client_id - dot_expander: + tag: dot_expander_meta_feature_category_c6043395 path: gitlab.production field: meta.feature_category - dot_expander: @@ -52,91 +60,113 @@ processors: tag: dot_expander_meta_gl_organization_id field: meta.organization_id - dot_expander: + tag: dot_expander_meta_remote_ip_9ccf505e path: gitlab.production field: meta.remote_ip - convert: + tag: convert_gitlab_production_meta_remote_ip_9972b4e2 field: gitlab.production.meta.remote_ip type: ip ignore_missing: true on_failure: - remove: + tag: remove_gitlab_production_meta_remote_ip_3b197ff0 field: gitlab.production.meta.remote_ip ignore_missing: true - append: + tag: append_error_message_9b833db1 field: error.message value: '{{{_ingest.on_failure_message}}}' - dot_expander: + tag: dot_expander_meta_user_b421f699 path: gitlab.production field: meta.user - dot_expander: + tag: dot_expander_meta_user_id_ed6d2d19 path: gitlab.production field: meta.user_id - dot_expander: + tag: dot_expander_meta_search_page_ed76c2f5 path: gitlab.production field: meta.search.page - rename: + tag: rename_gitlab_production_action_to_event_action_85fc0548 field: gitlab.production.action target_field: event.action ignore_missing: true - rename: + tag: rename_gitlab_production_meta_caller_id_to_event_provider_7abb1121 field: gitlab.production.meta.caller_id target_field: event.provider ignore_missing: true - rename: + tag: rename_gitlab_production_status_to_http_response_status_code_2457bda3 field: gitlab.production.status target_field: http.response.status_code ignore_missing: true - date: + tag: date_gitlab_production_time_to_timestamp_358364ab field: gitlab.production.time target_field: '@timestamp' formats: - yyyy-MM-dd'T'HH:mm:ss.SSSX if: ctx.gitlab.production.time != null - rename: + tag: rename_gitlab_production_pid_to_process_pid_e60d390f field: gitlab.production.pid target_field: process.pid ignore_missing: true - rename: + tag: rename_gitlab_production_worker_id_to_process_name_d53c09e4 field: gitlab.production.worker_id target_field: process.name ignore_missing: true - rename: + tag: rename_gitlab_production_correlation_id_to_event_id_bbbd6bef field: gitlab.production.correlation_id target_field: event.id ignore_missing: true - rename: + tag: rename_gitlab_production_duration_s_to_event_duration_e5b4f00e field: gitlab.production.duration_s target_field: event.duration ignore_missing: true - script: + tag: script_c2805a97 description: Convert event.duration from seconds to nanoseconds. if: ctx.event?.duration != null lang: painless source: | ctx.event['duration'] = ctx.event.duration * 1e9; - convert: + tag: convert_gitlab_production_remote_ip_to_source_ip_26fae183 target_field: source.ip field: gitlab.production.remote_ip type: ip ignore_missing: true ignore_failure: true - remove: + tag: remove_gitlab_production_remote_ip_7d0f8841 field: gitlab.production.remote_ip ignore_failure: true ignore_missing: true - rename: + tag: rename_gitlab_production_user_id_to_user_id_0868b653 field: gitlab.production.user_id target_field: user.id ignore_missing: true - convert: + tag: convert_user_id_c132c028 field: user.id type: string ignore_missing: true - rename: + tag: rename_gitlab_production_username_to_user_name_2072279a field: gitlab.production.username target_field: user.name ignore_missing: true - rename: + tag: rename_gitlab_production_ua_to_user_agent_original_ac9d5476 field: gitlab.production.ua target_field: user_agent.original ignore_missing: true @@ -236,62 +266,72 @@ processors: target_field: destination.as.organization.name ignore_missing: true - append: + tag: append_event_type_6adea021 field: event.type value: - info allow_duplicates: false - append: + tag: append_event_category_b9496563 field: event.category value: - database allow_duplicates: false if: ctx.url?.path == '/-/metrics' - append: + tag: append_event_category_d7694812 field: event.category value: - database allow_duplicates: false if: ctx.url?.path == '/dashboard/activity' - append: + tag: append_event_category_8f9cdd7e field: event.category value: - database allow_duplicates: false if: ctx.url?.path == '/api/graphql' - append: + tag: append_event_category_e705a42c field: event.category value: - web allow_duplicates: false if: ctx.url?.path == '/-/manifest.json' - append: + tag: append_event_category_7dd0bed7 field: event.category value: - authentication allow_duplicates: false if: ctx.url?.path == '/users/sign_in' - append: + tag: append_event_category_fa9d3f80 field: event.category value: - web allow_duplicates: false if: ctx.url?.path != null && ctx.url.path.contains("/search/") - append: + tag: append_event_category_d9fbfc11 field: event.category value: - database allow_duplicates: false if: ctx.url?.path == '/dashboard/groups' - convert: - field: gitlab.production.path_traversal_check_duration_s + field: gitlab.production.path_traversal_check_duration_s tag: convert_path_traversal_check_duration_s _to_double type: double ignore_missing: true on_failure: - remove: - field: gitlab.production.path_traversal_check_duration_s + tag: remove_gitlab_production_path_traversal_check_duration_s_4198423c + field: gitlab.production.path_traversal_check_duration_s ignore_missing: true - append: + tag: append_error_message_590a670f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -301,9 +341,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_production_db_ci_replica_txn_max_duration_s_776059d0 field: gitlab.production.db_ci_replica_txn_max_duration_s ignore_missing: true - append: + tag: append_error_message_5e3679e2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -313,9 +355,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_production_db_ci_replica_write_count_8443faea field: gitlab.production.db_ci_replica_write_count ignore_missing: true - append: + tag: append_error_message_7ccbefa7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -325,9 +369,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_production_db_ci_txn_max_duration_s_79a515a2 field: gitlab.production.db_ci_txn_max_duration_s ignore_missing: true - append: + tag: append_error_message_837ba9f9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -337,9 +383,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_production_db_ci_write_count_15f9ea80 field: gitlab.production.db_ci_write_count ignore_missing: true - append: + tag: append_error_message_9235d8f8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -349,9 +397,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_production_db_main_replica_txn_max_duration_s_462dfd02 field: gitlab.production.db_main_replica_txn_max_duration_s ignore_missing: true - append: + tag: append_error_message_c6ea7497 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -361,9 +411,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_production_db_main_replica_write_count_eaf4e6cc field: gitlab.production.db_main_replica_write_count ignore_missing: true - append: + tag: append_error_message_b7060e6e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -373,9 +425,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_production_db_main_txn_max_duration_s_4f28bc54 field: gitlab.production.db_main_txn_max_duration_s ignore_missing: true - append: + tag: append_error_message_2c441b58 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -385,9 +439,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_production_db_main_write_count_745dc0a2 field: gitlab.production.db_main_write_count ignore_missing: true - append: + tag: append_error_message_620e14cd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -397,9 +453,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_production_db_primary_txn_max_duration_s_c2fd25a2 field: gitlab.production.db_primary_txn_max_duration_s ignore_missing: true - append: + tag: append_error_message_76cc54e1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -409,9 +467,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_production_db_primary_write_count_508c54d0 field: gitlab.production.db_primary_write_count ignore_missing: true - append: + tag: append_error_message_f28f94d6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -421,9 +481,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_production_db_replica_txn_max_duration_s_eb5f85ea field: gitlab.production.db_replica_txn_max_duration_s ignore_missing: true - append: + tag: append_error_message_b6825d17 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -433,9 +495,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_production_db_replica_write_count_b6ee7350 field: gitlab.production.db_replica_write_count ignore_missing: true - append: + tag: append_error_message_2c1a4a94 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -445,9 +509,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_production_duration_s_dc7cf01a field: gitlab.production.duration_s ignore_missing: true - append: + tag: append_error_message_012c2395 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -457,9 +523,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_api_path_traversal_check_duration_s_a51d5f27 field: gitlab.api.path_traversal_check_duration_s ignore_missing: true - append: + tag: append_error_message_6b592b0b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -473,6 +541,7 @@ processors: copy_from: gitlab.production.meta.organization_id ignore_empty_value: true - script: + tag: script_a3eb5add description: Drops null/empty values recursively. lang: painless source: | @@ -492,8 +561,8 @@ processors: on_failure: - append: field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} - in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: field: event.kind value: pipeline_error diff --git a/packages/gitlab/data_stream/sidekiq/elasticsearch/ingest_pipeline/default.yml b/packages/gitlab/data_stream/sidekiq/elasticsearch/ingest_pipeline/default.yml index d54dc844459..8ff51ac9279 100644 --- a/packages/gitlab/data_stream/sidekiq/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gitlab/data_stream/sidekiq/elasticsearch/ingest_pipeline/default.yml @@ -2,6 +2,7 @@ description: Pipeline for processing sidekiq logs. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: 8.11.0 - rename: @@ -15,7 +16,8 @@ processors: ignore_missing: true tag: remove_message if: ctx.event?.original != null - - drop: + - drop: + tag: drop_a8ea228b if: ctx.event?.original?.startsWith('#') description: Drop if logline contains header(s), which startswith `#`. - json: @@ -23,12 +25,15 @@ processors: tag: 'json_decoding' target_field: gitlab.sidekiq - append: + tag: append_event_category_4595ee28 field: event.category value: web - append: + tag: append_event_type_8a66ccaa field: event.type value: info - set: + tag: set_event_kind_de80643c field: event.kind value: event - date: @@ -40,13 +45,16 @@ processors: tag: date_event_created_time_epoch on_failure: - append: + tag: append_error_message_f20e6e62 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' if: ctx.gitlab?.sidekiq?.time != null && ctx.gitlab.sidekiq.time != '' - remove: + tag: remove_gitlab_sidekiq_time_aceaf5fe field: gitlab.sidekiq.time ignore_missing: true - lowercase: + tag: lowercase_gitlab_sidekiq_severity_67779461 field: gitlab.sidekiq.severity ignore_missing: true - set: @@ -90,6 +98,7 @@ processors: if: ctx.gitlab?.sidekiq?.severity == 'debug' tag: set_severity_7 - remove: + tag: remove_gitlab_sidekiq_severity_0d8bfc04 field: gitlab.sidekiq.severity ignore_missing: true - date: @@ -102,28 +111,33 @@ processors: if: ctx.gitlab?.sidekiq?.created_at != null && ctx.gitlab.sidekiq.created_at != '' on_failure: - append: + tag: append_error_message_5c11dec6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: + tag: remove_gitlab_sidekiq_created_at_325ffd25 field: gitlab.sidekiq.created_at ignore_missing: true - date: field: gitlab.sidekiq.enqueued_at formats: - "ISO8601" - target_field: gitlab.sidekiq.enqueued_at + target_field: gitlab.sidekiq.enqueued_at timezone: "UTC" tag: enqueued_date_event_created_time_epoch if: ctx.gitlab?.sidekiq?.enqueued_at != null && ctx.gitlab.sidekiq.enqueued_at != '' on_failure: - append: + tag: append_error_message_bce42ce3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - - rename: + - rename: + tag: rename_gitlab_sidekiq_pid_to_process_pid_02394ec6 field: gitlab.sidekiq.pid target_field: process.pid ignore_missing: true - - rename: + - rename: + tag: rename_gitlab_sidekiq_message_to_message_e488ac2f field: gitlab.sidekiq.message target_field: message ignore_missing: true @@ -133,9 +147,11 @@ processors: target_field: gitlab.sidekiq.trace_propagation_headers.sentry_trace ignore_missing: true - remove: + tag: remove_gitlab_sidekiq_trace_propagation_headers_sentry-trace_70b8f2e7 field: gitlab.sidekiq.trace_propagation_headers.sentry-trace ignore_missing: true - - rename: + - rename: + tag: rename_gitlab_sidekiq_duration_to_event_duration_23ebc64d field: gitlab.sidekiq.duration target_field: event.duration ignore_missing: true @@ -154,24 +170,30 @@ processors: if: ctx.gitlab?.sidekiq?.completed_at != null && ctx.gitlab.sidekiq.completed_at != '' on_failure: - append: + tag: append_error_message_e0cdca6f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: + tag: remove_gitlab_sidekiq_completed_at_abc7e6f4 field: gitlab.sidekiq.completed_at ignore_missing: true - rename: + tag: rename_gitlab_sidekiq_db_duration_to_gitlab_sidekiq_db_duration_m_5e2023b5 field: gitlab.sidekiq.db_duration target_field: gitlab.sidekiq.db.duration_m ignore_missing: true - rename: + tag: rename_gitlab_sidekiq_db_duration_s_to_gitlab_sidekiq_db_duration_s_47cd0515 field: gitlab.sidekiq.db_duration_s target_field: gitlab.sidekiq.db.duration_s ignore_missing: true - rename: + tag: rename_gitlab_sidekiq_gitaly_duration_to_gitlab_gitaly_duration_e10280e9 field: gitlab.sidekiq.gitaly_duration target_field: gitlab.gitaly.duration ignore_missing: true - - rename: + - rename: + tag: rename_gitlab_sidekiq_gitaly_calls_to_gitlab_gitaly_calls_f0ac9cb7 field: gitlab.sidekiq.gitaly_calls target_field: gitlab.gitaly.calls ignore_missing: true @@ -182,9 +204,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_cpu_s_1cc5ab61 field: gitlab.sidekiq.cpu_s ignore_missing: true - append: + tag: append_error_message_718c570f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -194,9 +218,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_cache_duration_s_4c5fc387 field: gitlab.sidekiq.redis_cache_duration_s ignore_missing: true - append: + tag: append_error_message_b2afd6b8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -206,9 +232,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_cached_count_c2831e25 field: gitlab.sidekiq.db_cached_count ignore_missing: true - append: + tag: append_error_message_1b167f48 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -218,9 +246,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_ci_cached_count_43ee2b47 field: gitlab.sidekiq.db_ci_cached_count ignore_missing: true - append: + tag: append_error_message_4801834f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -230,9 +260,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_ci_count_2fdcca15 field: gitlab.sidekiq.db_ci_count ignore_missing: true - append: + tag: append_error_message_3c5c26da field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -242,9 +274,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_ci_duration_s_404592fb field: gitlab.sidekiq.db_ci_duration_s ignore_missing: true - append: + tag: append_error_message_59dfbbb3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -254,9 +288,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_ci_replica_cached_count_40b3128b field: gitlab.sidekiq.db_ci_replica_cached_count ignore_missing: true - append: + tag: append_error_message_e0d86eb2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -266,9 +302,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_ci_replica_count_6d537749 field: gitlab.sidekiq.db_ci_replica_count ignore_missing: true - append: + tag: append_error_message_4a5f7a33 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -278,9 +316,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_ci_replica_duration_s_06855477 field: gitlab.sidekiq.db_ci_replica_duration_s ignore_missing: true - append: + tag: append_error_message_278d942e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -290,9 +330,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_ci_replica_txn_count_939193dd field: gitlab.sidekiq.db_ci_replica_txn_count ignore_missing: true - append: + tag: append_error_message_645f0154 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -302,9 +344,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_ci_replica_txn_duration_s_b26a363b field: gitlab.sidekiq.db_ci_replica_txn_duration_s ignore_missing: true - append: + tag: append_error_message_dc264125 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -314,9 +358,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_ci_replica_txn_max_duration_s_2d8ab557 field: gitlab.sidekiq.db_ci_replica_txn_max_duration_s ignore_missing: true - append: + tag: append_error_message_62a90ede field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -326,9 +372,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_ci_replica_wal_cached_count_3d17c1eb field: gitlab.sidekiq.db_ci_replica_wal_cached_count ignore_missing: true - append: + tag: append_error_message_a93f9c6f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -338,9 +386,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_ci_replica_wal_count_71ed1829 field: gitlab.sidekiq.db_ci_replica_wal_count ignore_missing: true - append: + tag: append_error_message_a7195746 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -350,9 +400,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_ci_replica_write_count_30b8fe61 field: gitlab.sidekiq.db_ci_replica_write_count ignore_missing: true - append: + tag: append_error_message_a662a827 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -362,9 +414,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_ci_txn_count_c1c36971 field: gitlab.sidekiq.db_ci_txn_count ignore_missing: true - append: + tag: append_error_message_84dd199d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -374,9 +428,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_ci_txn_duration_s_82a697d7 field: gitlab.sidekiq.db_ci_txn_duration_s ignore_missing: true - append: + tag: append_error_message_9feb8f58 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -386,9 +442,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_ci_txn_max_duration_s_52a2d70b field: gitlab.sidekiq.db_ci_txn_max_duration_s ignore_missing: true - append: + tag: append_error_message_670f4f0b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -398,9 +456,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_ci_wal_cached_count_68ce08f7 field: gitlab.sidekiq.db_ci_wal_cached_count ignore_missing: true - append: + tag: append_error_message_634fa516 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -410,9 +470,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_ci_wal_count_8bdb9ead field: gitlab.sidekiq.db_ci_wal_count ignore_missing: true - append: + tag: append_error_message_825b7fbb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -422,9 +484,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_ci_write_count_2c67f8e5 field: gitlab.sidekiq.db_ci_write_count ignore_missing: true - append: + tag: append_error_message_8861de06 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -434,9 +498,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_count_eec6e7f7 field: gitlab.sidekiq.db_count ignore_missing: true - append: + tag: append_error_message_6615dbcb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -446,9 +512,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_main_cached_count_a2cbb1b3 field: gitlab.sidekiq.db_main_cached_count ignore_missing: true - append: + tag: append_error_message_54c2451c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -458,9 +526,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_main_count_9f061719 field: gitlab.sidekiq.db_main_count ignore_missing: true - append: + tag: append_error_message_2cdafe9d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -470,9 +540,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_main_duration_s_17aefc67 field: gitlab.sidekiq.db_main_duration_s ignore_missing: true - append: + tag: append_error_message_fc34a370 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -482,9 +554,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_main_replica_cached_count_f53e90af field: gitlab.sidekiq.db_main_replica_cached_count ignore_missing: true - append: + tag: append_error_message_cbadd6e9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -494,9 +568,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_main_replica_count_015360c5 field: gitlab.sidekiq.db_main_replica_count ignore_missing: true - append: + tag: append_error_message_e9689b2c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -506,9 +582,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_main_replica_duration_s_5d895e3b field: gitlab.sidekiq.db_main_replica_duration_s ignore_missing: true - append: + tag: append_error_message_c8b7daa5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -518,9 +596,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_main_replica_txn_count_5d454039 field: gitlab.sidekiq.db_main_replica_txn_count ignore_missing: true - append: + tag: append_error_message_a1316bbf field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -530,9 +610,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_main_replica_txn_duration_s_3ad6caaf field: gitlab.sidekiq.db_main_replica_txn_duration_s ignore_missing: true - append: + tag: append_error_message_9b3f271a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -542,9 +624,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_main_replica_txn_max_duration_s_fe55a5b3 field: gitlab.sidekiq.db_main_replica_txn_max_duration_s ignore_missing: true - append: + tag: append_error_message_4e0e9d85 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -554,9 +638,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_main_replica_wal_cached_count_d5305cf7 field: gitlab.sidekiq.db_main_replica_wal_cached_count ignore_missing: true - append: + tag: append_error_message_e146ebac field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -566,9 +652,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_main_replica_wal_count_2ff43965 field: gitlab.sidekiq.db_main_replica_wal_count ignore_missing: true - append: + tag: append_error_message_3cfa8989 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -578,9 +666,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_main_replica_write_count_fcf4a55d field: gitlab.sidekiq.db_main_replica_write_count ignore_missing: true - append: + tag: append_error_message_e66a4770 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -590,9 +680,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_main_txn_count_7ecaf0d5 field: gitlab.sidekiq.db_main_txn_count ignore_missing: true - append: + tag: append_error_message_a00fadbe field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -602,9 +694,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_main_txn_duration_s_96114273 field: gitlab.sidekiq.db_main_txn_duration_s ignore_missing: true - append: + tag: append_error_message_12787027 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -614,9 +708,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_main_txn_max_duration_s_fe4f5dff field: gitlab.sidekiq.db_main_txn_max_duration_s ignore_missing: true - append: + tag: append_error_message_3fd04108 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -626,9 +722,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_main_wal_cached_count_c694695b field: gitlab.sidekiq.db_main_wal_cached_count ignore_missing: true - append: + tag: append_error_message_f7d6f0fd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -638,9 +736,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_main_wal_count_37e32231 field: gitlab.sidekiq.db_main_wal_count ignore_missing: true - append: + tag: append_error_message_2023f704 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -650,9 +750,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_main_write_count_13641829 field: gitlab.sidekiq.db_main_write_count ignore_missing: true - append: + tag: append_error_message_7559b9b9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -662,9 +764,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_primary_cached_count_c9868a21 field: gitlab.sidekiq.db_primary_cached_count ignore_missing: true - append: + tag: append_error_message_5d6d5d81 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -674,9 +778,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_primary_count_2b2e91ab field: gitlab.sidekiq.db_primary_count ignore_missing: true - append: + tag: append_error_message_a56175c2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -686,9 +792,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_primary_duration_s_ab352641 field: gitlab.sidekiq.db_primary_duration_s ignore_missing: true - append: + tag: append_error_message_15168a91 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -698,9 +806,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_primary_txn_count_20d76f33 field: gitlab.sidekiq.db_primary_txn_count ignore_missing: true - append: + tag: append_error_message_83ed15c7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -710,9 +820,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_primary_txn_duration_s_bee3c399 field: gitlab.sidekiq.db_primary_txn_duration_s ignore_missing: true - append: + tag: append_error_message_9d43b1d4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -722,9 +834,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_primary_txn_max_duration_s_1a2e8915 field: gitlab.sidekiq.db_primary_txn_max_duration_s ignore_missing: true - append: + tag: append_error_message_a924a7dd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -734,9 +848,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_primary_wal_cached_count_1350ea1d field: gitlab.sidekiq.db_primary_wal_cached_count ignore_missing: true - append: + tag: append_error_message_2ad9150e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -746,9 +862,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_primary_wal_count_e5014d97 field: gitlab.sidekiq.db_primary_wal_count ignore_missing: true - append: + tag: append_error_message_2dbd7fbd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -758,9 +876,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_primary_write_count_ac363c47 field: gitlab.sidekiq.db_primary_write_count ignore_missing: true - append: + tag: append_error_message_b2adbada field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -770,9 +890,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_replica_cached_count_dac72219 field: gitlab.sidekiq.db_replica_cached_count ignore_missing: true - append: + tag: append_error_message_885535e7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -782,9 +904,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_replica_count_fa43e07b field: gitlab.sidekiq.db_replica_count ignore_missing: true - append: + tag: append_error_message_30c539d8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -794,9 +918,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_replica_duration_s_3e1cc861 field: gitlab.sidekiq.db_replica_duration_s ignore_missing: true - append: + tag: append_error_message_7d1afdcf field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -806,9 +932,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_replica_txn_count_16bc21c3 field: gitlab.sidekiq.db_replica_txn_count ignore_missing: true - append: + tag: append_error_message_1fabc005 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -818,9 +946,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_replica_txn_duration_s_69ee64d1 field: gitlab.sidekiq.db_replica_txn_duration_s ignore_missing: true - append: + tag: append_error_message_dd4af432 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -830,9 +960,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_replica_txn_max_duration_s_ad09ff6d field: gitlab.sidekiq.db_replica_txn_max_duration_s ignore_missing: true - append: + tag: append_error_message_4e1f25e3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -842,9 +974,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_replica_wal_cached_count_dd659e2d field: gitlab.sidekiq.db_replica_wal_cached_count ignore_missing: true - append: + tag: append_error_message_92dbad24 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -854,9 +988,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_replica_wal_count_71d91bbf field: gitlab.sidekiq.db_replica_wal_count ignore_missing: true - append: + tag: append_error_message_3b0f6a83 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -866,9 +1002,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_replica_write_count_799d1eaf field: gitlab.sidekiq.db_replica_write_count ignore_missing: true - append: + tag: append_error_message_e5029c20 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -878,9 +1016,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_txn_count_301a14b7 field: gitlab.sidekiq.db_txn_count ignore_missing: true - append: + tag: append_error_message_2346a4aa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -890,9 +1030,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_db_write_count_95be0deb field: gitlab.sidekiq.db_write_count ignore_missing: true - append: + tag: append_error_message_84c0c18b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -902,9 +1044,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_dead_b8618fbf field: gitlab.sidekiq.dead ignore_missing: true - append: + tag: append_error_message_32e6f1ef field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -914,9 +1058,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_duration_s_4fee092f field: gitlab.sidekiq.duration_s ignore_missing: true - append: + tag: append_error_message_f7f02387 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -926,9 +1072,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_enqueue_latency_s_88732f15 field: gitlab.sidekiq.enqueue_latency_s ignore_missing: true - append: + tag: append_error_message_0587ea54 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -938,9 +1086,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_exclusive_lock_hold_duration_s_4d69c4bf field: gitlab.sidekiq.exclusive_lock_hold_duration_s ignore_missing: true - append: + tag: append_error_message_77ef8c90 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -950,9 +1100,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_exclusive_lock_requested_count_27e93783 field: gitlab.sidekiq.exclusive_lock_requested_count ignore_missing: true - append: + tag: append_error_message_e41c921c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -962,9 +1114,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_exclusive_lock_wait_duration_s_e8dd035b field: gitlab.sidekiq.exclusive_lock_wait_duration_s ignore_missing: true - append: + tag: append_error_message_86576700 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -974,9 +1128,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_batched_git_ref_updates_cleanup_scheduler_worker_stats_total_projects_e38646a2 field: gitlab.sidekiq.extra.batched_git_ref_updates_cleanup_scheduler_worker.stats.total_projects ignore_missing: true - append: + tag: append_error_message_2dc7323b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -986,9 +1142,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_cells_stale_requests_cleanup_cron_worker_removed_count_861ebc2d field: gitlab.sidekiq.extra.cells_stale_requests_cleanup_cron_worker.removed_count ignore_missing: true - append: + tag: append_error_message_03fe26f8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -998,9 +1156,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_ci_catalog_resources_process_sync_events_worker_consumable_events_a8cd6c69 field: gitlab.sidekiq.extra.ci_catalog_resources_process_sync_events_worker.consumable_events ignore_missing: true - append: + tag: append_error_message_23b0fff1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1010,9 +1170,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_ci_catalog_resources_process_sync_events_worker_estimated_total_events_cd0d30eb field: gitlab.sidekiq.extra.ci_catalog_resources_process_sync_events_worker.estimated_total_events ignore_missing: true - append: + tag: append_error_message_61c18f1d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1022,9 +1184,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_ci_pipeline_artifacts_expire_artifacts_worker_destroyed_pipeline_artifacts_count_b9d18dbf field: gitlab.sidekiq.extra.ci_pipeline_artifacts_expire_artifacts_worker.destroyed_pipeline_artifacts_count ignore_missing: true - append: + tag: append_error_message_a6c6cce7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1034,9 +1198,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_ci_unlock_pipelines_in_queue_worker_remaining_pending_5dca21fd field: gitlab.sidekiq.extra.ci_unlock_pipelines_in_queue_worker.remaining_pending ignore_missing: true - append: + tag: append_error_message_94e0c0b9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1046,9 +1212,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_ci_update_locked_unknown_artifacts_worker_locked_count_9984992b field: gitlab.sidekiq.extra.ci_update_locked_unknown_artifacts_worker.locked_count ignore_missing: true - append: + tag: append_error_message_69d63a72 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1058,9 +1226,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_ci_update_locked_unknown_artifacts_worker_removed_count_9718525d field: gitlab.sidekiq.extra.ci_update_locked_unknown_artifacts_worker.removed_count ignore_missing: true - append: + tag: append_error_message_8e5b676e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1070,9 +1240,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_container_expiration_policy_worker_cleanup_required_count_d54e5867 field: gitlab.sidekiq.extra.container_expiration_policy_worker.cleanup_required_count ignore_missing: true - append: + tag: append_error_message_b0e27d18 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1082,9 +1254,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_container_expiration_policy_worker_cleanup_total_count_75d9fe69 field: gitlab.sidekiq.extra.container_expiration_policy_worker.cleanup_total_count ignore_missing: true - append: + tag: append_error_message_a00c5fb7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1094,9 +1268,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_container_expiration_policy_worker_cleanup_unfinished_count_5eeeac77 field: gitlab.sidekiq.extra.container_expiration_policy_worker.cleanup_unfinished_count ignore_missing: true - append: + tag: append_error_message_aa31ca34 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1106,9 +1282,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_expire_build_artifacts_worker_destroyed_job_artifacts_count_3599d4e1 field: gitlab.sidekiq.extra.expire_build_artifacts_worker.destroyed_job_artifacts_count ignore_missing: true - append: + tag: append_error_message_978e4941 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1118,9 +1296,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_loose_foreign_keys_ci_pipelines_builds_cleanup_cron_worker_stats_delete_count_4ec650fa field: gitlab.sidekiq.extra.loose_foreign_keys_ci_pipelines_builds_cleanup_cron_worker.stats.delete_count ignore_missing: true - append: + tag: append_error_message_a9fa3420 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1130,9 +1310,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_loose_foreign_keys_ci_pipelines_builds_cleanup_cron_worker_stats_over_limit_cb1afc6e field: gitlab.sidekiq.extra.loose_foreign_keys_ci_pipelines_builds_cleanup_cron_worker.stats.over_limit ignore_missing: true - append: + tag: append_error_message_0ac2921b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1142,9 +1324,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_loose_foreign_keys_ci_pipelines_builds_cleanup_cron_worker_stats_turbo_mode_a06c1ed6 field: gitlab.sidekiq.extra.loose_foreign_keys_ci_pipelines_builds_cleanup_cron_worker.stats.turbo_mode ignore_missing: true - append: + tag: append_error_message_1b09d95b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1154,9 +1338,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_loose_foreign_keys_ci_pipelines_builds_cleanup_cron_worker_stats_update_count_20f86d32 field: gitlab.sidekiq.extra.loose_foreign_keys_ci_pipelines_builds_cleanup_cron_worker.stats.update_count ignore_missing: true - append: + tag: append_error_message_4a959126 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1166,9 +1352,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_loose_foreign_keys_cleanup_worker_stats_delete_count_73b15862 field: gitlab.sidekiq.extra.loose_foreign_keys_cleanup_worker.stats.delete_count ignore_missing: true - append: + tag: append_error_message_f41fd592 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1178,9 +1366,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_loose_foreign_keys_cleanup_worker_stats_over_limit_93909176 field: gitlab.sidekiq.extra.loose_foreign_keys_cleanup_worker.stats.over_limit ignore_missing: true - append: + tag: append_error_message_8942d70f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1190,9 +1380,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_loose_foreign_keys_cleanup_worker_stats_turbo_mode_996d8066 field: gitlab.sidekiq.extra.loose_foreign_keys_cleanup_worker.stats.turbo_mode ignore_missing: true - append: + tag: append_error_message_625b90bf field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1202,9 +1394,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_loose_foreign_keys_cleanup_worker_stats_update_count_0a06c252 field: gitlab.sidekiq.extra.loose_foreign_keys_cleanup_worker.stats.update_count ignore_missing: true - append: + tag: append_error_message_393ee028 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1214,9 +1408,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_loose_foreign_keys_merge_request_diff_commit_cleanup_worker_stats_delete_count_f003d42a field: gitlab.sidekiq.extra.loose_foreign_keys_merge_request_diff_commit_cleanup_worker.stats.delete_count ignore_missing: true - append: + tag: append_error_message_1fba4f95 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1226,9 +1422,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_loose_foreign_keys_merge_request_diff_commit_cleanup_worker_stats_over_limit_a0d67406 field: gitlab.sidekiq.extra.loose_foreign_keys_merge_request_diff_commit_cleanup_worker.stats.over_limit ignore_missing: true - append: + tag: append_error_message_462da28c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1238,9 +1436,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_loose_foreign_keys_merge_request_diff_commit_cleanup_worker_stats_update_count_ec1c1c92 field: gitlab.sidekiq.extra.loose_foreign_keys_merge_request_diff_commit_cleanup_worker.stats.update_count ignore_missing: true - append: + tag: append_error_message_f2a70023 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1250,9 +1450,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_namespaces_process_sync_events_worker_consumable_events_d39e8441 field: gitlab.sidekiq.extra.namespaces_process_sync_events_worker.consumable_events ignore_missing: true - append: + tag: append_error_message_749c4b6b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1262,9 +1464,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_namespaces_process_sync_events_worker_estimated_total_events_359e3b9b field: gitlab.sidekiq.extra.namespaces_process_sync_events_worker.estimated_total_events ignore_missing: true - append: + tag: append_error_message_893340f7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1274,9 +1478,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_namespaces_process_sync_events_worker_processed_events_3a0d19bf field: gitlab.sidekiq.extra.namespaces_process_sync_events_worker.processed_events ignore_missing: true - append: + tag: append_error_message_c09dacae field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1286,9 +1492,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_object_storage_delete_stale_direct_uploads_worker_execution_timeout_a711cad5 field: gitlab.sidekiq.extra.object_storage_delete_stale_direct_uploads_worker.execution_timeout ignore_missing: true - append: + tag: append_error_message_e12074ae field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1298,9 +1506,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_object_storage_delete_stale_direct_uploads_worker_total_deleted_stale_entries_e70207f5 field: gitlab.sidekiq.extra.object_storage_delete_stale_direct_uploads_worker.total_deleted_stale_entries ignore_missing: true - append: + tag: append_error_message_5ec0aa07 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1310,9 +1520,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_object_storage_delete_stale_direct_uploads_worker_total_pending_entries_ffca9215 field: gitlab.sidekiq.extra.object_storage_delete_stale_direct_uploads_worker.total_pending_entries ignore_missing: true - append: + tag: append_error_message_8da79c2f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1322,9 +1534,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_packages_cleanup_delete_orphaned_dependencies_worker_deleted_rows_count_f0618bbb field: gitlab.sidekiq.extra.packages_cleanup_delete_orphaned_dependencies_worker.deleted_rows_count ignore_missing: true - append: + tag: append_error_message_e965d54b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1334,9 +1548,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_packages_cleanup_delete_orphaned_dependencies_worker_last_processed_packages_dependency_id_57449541 field: gitlab.sidekiq.extra.packages_cleanup_delete_orphaned_dependencies_worker.last_processed_packages_dependency_id ignore_missing: true - append: + tag: append_error_message_d43a1f5d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1346,9 +1562,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_pages_deactivate_expired_deployments_cron_worker_deactivate_expired_pages_deployments_deactivated_deployments_9a4a3b68 field: gitlab.sidekiq.extra.pages_deactivate_expired_deployments_cron_worker.deactivate_expired_pages_deployments.deactivated_deployments ignore_missing: true - append: + tag: append_error_message_d8f5c10b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1358,9 +1576,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_pages_deactivate_expired_deployments_cron_worker_deactivate_expired_pages_deployments_duration_3cbab1d4 field: gitlab.sidekiq.extra.pages_deactivate_expired_deployments_cron_worker.deactivate_expired_pages_deployments.duration ignore_missing: true - append: + tag: append_error_message_0f9b1ab4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1370,9 +1590,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_projects_process_sync_events_worker_consumable_events_fbcacfb1 field: gitlab.sidekiq.extra.projects_process_sync_events_worker.consumable_events ignore_missing: true - append: + tag: append_error_message_010f32e5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1382,9 +1604,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_projects_process_sync_events_worker_estimated_total_events_c1222203 field: gitlab.sidekiq.extra.projects_process_sync_events_worker.estimated_total_events ignore_missing: true - append: + tag: append_error_message_bdd03d3d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1394,9 +1618,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_extra_projects_process_sync_events_worker_processed_events_5e16350f field: gitlab.sidekiq.extra.projects_process_sync_events_worker.processed_events ignore_missing: true - append: + tag: append_error_message_0ef2f5a4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1406,9 +1632,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_job_size_bytes_8ec9fd27 field: gitlab.sidekiq.job_size_bytes ignore_missing: true - append: + tag: append_error_message_deb866a4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1418,9 +1646,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_mem_bytes_1b0696f1 field: gitlab.sidekiq.mem_bytes ignore_missing: true - append: + tag: append_error_message_8e83375c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1430,9 +1660,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_mem_mallocs_69fb54a9 field: gitlab.sidekiq.mem_mallocs ignore_missing: true - append: + tag: append_error_message_25cb8d76 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1442,9 +1674,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_mem_objects_1195a529 field: gitlab.sidekiq.mem_objects ignore_missing: true - append: + tag: append_error_message_32a2f06d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1454,9 +1688,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_mem_total_bytes_5d6fc9b9 field: gitlab.sidekiq.mem_total_bytes ignore_missing: true - append: + tag: append_error_message_0e5b4b8b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1467,9 +1703,11 @@ processors: if: ctx.gitlab?.sidekiq?.meta?.remote_ip != '' on_failure: - remove: + tag: remove_gitlab_sidekiq_meta_remote_ip_34f5d6d1 field: gitlab.sidekiq.meta.remote_ip ignore_missing: true - append: + tag: append_error_message_2a9fa8be field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1479,9 +1717,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_queue_duration_s_a854876b field: gitlab.sidekiq.queue_duration_s ignore_missing: true - append: + tag: append_error_message_1e7c38a7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1491,9 +1731,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_cache_calls_53931685 field: gitlab.sidekiq.redis_cache_calls ignore_missing: true - append: + tag: append_error_message_c544693f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1503,9 +1745,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_cache_read_bytes_6dc2f64b field: gitlab.sidekiq.redis_cache_read_bytes ignore_missing: true - append: + tag: append_error_message_f48b39de field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1515,9 +1759,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_cache_write_bytes_dc5463d9 field: gitlab.sidekiq.redis_cache_write_bytes ignore_missing: true - append: + tag: append_error_message_ce0c8d01 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1527,9 +1773,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_calls_159990f1 field: gitlab.sidekiq.redis_calls ignore_missing: true - append: + tag: append_error_message_2e643878 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1539,9 +1787,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_duration_s_038b0c2b field: gitlab.sidekiq.redis_duration_s ignore_missing: true - append: + tag: append_error_message_f9d9074b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1551,9 +1801,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_feature_flag_calls_a1d660bb field: gitlab.sidekiq.redis_feature_flag_calls ignore_missing: true - append: + tag: append_error_message_bbceffd2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1563,9 +1815,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_feature_flag_duration_s_4b354da9 field: gitlab.sidekiq.redis_feature_flag_duration_s ignore_missing: true - append: + tag: append_error_message_e0fda083 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1575,9 +1829,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_feature_flag_read_bytes_e7ac8dc9 field: gitlab.sidekiq.redis_feature_flag_read_bytes ignore_missing: true - append: + tag: append_error_message_ce2f119d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1587,9 +1843,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_feature_flag_write_bytes_d2cd678b field: gitlab.sidekiq.redis_feature_flag_write_bytes ignore_missing: true - append: + tag: append_error_message_aeb7f030 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1599,9 +1857,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_queues_calls_716d1dcf field: gitlab.sidekiq.redis_queues_calls ignore_missing: true - append: + tag: append_error_message_5e6e04bb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1611,9 +1871,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_queues_duration_s_2ba5263d field: gitlab.sidekiq.redis_queues_duration_s ignore_missing: true - append: + tag: append_error_message_91426e46 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1623,9 +1885,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_queues_metadata_calls_41cb4d75 field: gitlab.sidekiq.redis_queues_metadata_calls ignore_missing: true - append: + tag: append_error_message_19947ef1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1635,9 +1899,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_queues_metadata_duration_s_2782f8ef field: gitlab.sidekiq.redis_queues_metadata_duration_s ignore_missing: true - append: + tag: append_error_message_032a30aa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1647,9 +1913,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_queues_metadata_read_bytes_6d85a15b field: gitlab.sidekiq.redis_queues_metadata_read_bytes ignore_missing: true - append: + tag: append_error_message_32094228 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1659,9 +1927,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_queues_metadata_write_bytes_cf1ea551 field: gitlab.sidekiq.redis_queues_metadata_write_bytes ignore_missing: true - append: + tag: append_error_message_dea1e12b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1671,9 +1941,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_queues_read_bytes_0717e065 field: gitlab.sidekiq.redis_queues_read_bytes ignore_missing: true - append: + tag: append_error_message_82a02658 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1683,9 +1955,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_queues_write_bytes_ae06795f field: gitlab.sidekiq.redis_queues_write_bytes ignore_missing: true - append: + tag: append_error_message_0c5bb8b9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1695,9 +1969,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_rate_limiting_calls_c9a3b251 field: gitlab.sidekiq.redis_rate_limiting_calls ignore_missing: true - append: + tag: append_error_message_5e22d22f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1707,9 +1983,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_rate_limiting_duration_s_810153bb field: gitlab.sidekiq.redis_rate_limiting_duration_s ignore_missing: true - append: + tag: append_error_message_145ea8ac field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1719,9 +1997,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_rate_limiting_write_bytes_cd7a11bd field: gitlab.sidekiq.redis_rate_limiting_write_bytes ignore_missing: true - append: + tag: append_error_message_46d24bc1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1731,9 +2011,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_read_bytes_82ec7527 field: gitlab.sidekiq.redis_read_bytes ignore_missing: true - append: + tag: append_error_message_9d2f188d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1743,9 +2025,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_shared_state_calls_e624888b field: gitlab.sidekiq.redis_shared_state_calls ignore_missing: true - append: + tag: append_error_message_bb017cec field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1755,9 +2039,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_shared_state_duration_s_4accced9 field: gitlab.sidekiq.redis_shared_state_duration_s ignore_missing: true - append: + tag: append_error_message_2f1ad8f5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1767,9 +2053,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_shared_state_read_bytes_a6064889 field: gitlab.sidekiq.redis_shared_state_read_bytes ignore_missing: true - append: + tag: append_error_message_68ce0c63 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1779,9 +2067,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_shared_state_write_bytes_263f91eb field: gitlab.sidekiq.redis_shared_state_write_bytes ignore_missing: true - append: + tag: append_error_message_b9892e2e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1791,9 +2081,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_redis_write_bytes_8fa9a225 field: gitlab.sidekiq.redis_write_bytes ignore_missing: true - append: + tag: append_error_message_86358076 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -1805,9 +2097,11 @@ processors: if: ctx.gitlab?.sidekiq?.retried_at != null && ctx.gitlab.sidekiq.retried_at != '' on_failure: - remove: + tag: remove_gitlab_sidekiq_retried_at_125add73 field: gitlab.sidekiq.retried_at ignore_missing: true - append: + tag: append_error_message_1e46f436 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1831,9 +2125,11 @@ processors: if: ctx.gitlab?.sidekiq?.retry != null && ctx.gitlab.sidekiq.retry != '' on_failure: - remove: + tag: remove_gitlab_sidekiq_retry_0bb0fbaf field: gitlab.sidekiq.retry ignore_missing: true - append: + tag: append_error_message_b0c58117 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1843,9 +2139,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_retry_count_5b6543c1 field: gitlab.sidekiq.retry_count ignore_missing: true - append: + tag: append_error_message_b909d805 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -1857,9 +2155,11 @@ processors: if: ctx.gitlab?.sidekiq?.scheduled_at != null && ctx.gitlab.sidekiq.scheduled_at != '' on_failure: - remove: + tag: remove_gitlab_sidekiq_scheduled_at_e334cfd3 field: gitlab.sidekiq.scheduled_at ignore_missing: true - append: + tag: append_error_message_660a1656 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1869,9 +2169,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_scheduling_latency_s_d8ac0fc7 field: gitlab.sidekiq.scheduling_latency_s ignore_missing: true - append: + tag: append_error_message_0143cc2c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1881,9 +2183,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_status_expiration_f051938d field: gitlab.sidekiq.status_expiration ignore_missing: true - append: + tag: append_error_message_4e4f5259 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1893,9 +2197,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_target_duration_s_ad607e2d field: gitlab.sidekiq.target_duration_s ignore_missing: true - append: + tag: append_error_message_18854f6f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1905,9 +2211,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_target_scheduling_latency_s_76e5c059 field: gitlab.sidekiq.target_scheduling_latency_s ignore_missing: true - append: + tag: append_error_message_905addca field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1917,9 +2225,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_gitlab_sidekiq_version_2211b1b5 field: gitlab.sidekiq.version ignore_missing: true - append: + tag: append_error_message_eac0e71b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -1941,11 +2251,13 @@ processors: } drop(ctx); - append: + tag: append_event_kind_d2f25e35 field: event.kind value: pipeline_error allow_duplicates: false if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/gitlab/manifest.yml b/packages/gitlab/manifest.yml index d6fb5b11605..39d98936e8d 100644 --- a/packages/gitlab/manifest.yml +++ b/packages/gitlab/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.1.3 name: gitlab title: GitLab -version: "2.6.1" +version: "2.6.2" description: Collect logs from GitLab with Elastic Agent. type: integration categories: diff --git a/packages/google_scc/changelog.yml b/packages/google_scc/changelog.yml index 7653a08470b..960b8702730 100644 --- a/packages/google_scc/changelog.yml +++ b/packages/google_scc/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.6.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "2.6.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/google_scc/data_stream/asset/elasticsearch/ingest_pipeline/default.yml b/packages/google_scc/data_stream/asset/elasticsearch/ingest_pipeline/default.yml index 59571c0b9e2..db4a993d41c 100644 --- a/packages/google_scc/data_stream/asset/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_scc/data_stream/asset/elasticsearch/ingest_pipeline/default.yml @@ -35,6 +35,7 @@ processors: ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: 'ctx.event?.original != null' @@ -45,6 +46,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_8075efaa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - drop: @@ -70,6 +72,7 @@ processors: if: ctx.json?.window?.startTime != null && ctx.json.window.startTime != '' on_failure: - append: + tag: append_error_message_9993cb1a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - pipeline: @@ -120,7 +123,7 @@ processors: ignore_empty_value: true - set: field: message - tag: set_message + tag: set_message_7c33cbce copy_from: google_scc.asset.service_perimeter.description ignore_empty_value: true - set: @@ -184,6 +187,7 @@ processors: tag: set_pipeline_error_into_event_kind if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/google_scc/data_stream/asset/elasticsearch/ingest_pipeline/pipeline_asset.yml b/packages/google_scc/data_stream/asset/elasticsearch/ingest_pipeline/pipeline_asset.yml index 41fbb8b250e..ff47c1375d3 100644 --- a/packages/google_scc/data_stream/asset/elasticsearch/ingest_pipeline/pipeline_asset.yml +++ b/packages/google_scc/data_stream/asset/elasticsearch/ingest_pipeline/pipeline_asset.yml @@ -56,6 +56,7 @@ processors: tag: rename_asset_accessLevel_custom_expr ignore_missing: true - foreach: + tag: foreach_json_asset_accessLevel_basic_conditions_05eb13a5 field: json.asset.accessLevel.basic.conditions if: ctx.json?.asset?.accessLevel?.basic?.conditions instanceof List ignore_failure: true @@ -76,6 +77,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_asset_accessLevel_basic_conditions_431126c0 field: json.asset.accessLevel.basic.conditions if: ctx.json?.asset?.accessLevel?.basic?.conditions instanceof List ignore_failure: true @@ -92,6 +94,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_asset_accessLevel_basic_conditions_29bb0707 field: json.asset.accessLevel.basic.conditions if: ctx.json?.asset?.accessLevel?.basic?.conditions instanceof List ignore_failure: true @@ -108,6 +111,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_asset_accessLevel_basic_conditions_85b9440f field: json.asset.accessLevel.basic.conditions if: ctx.json?.asset?.accessLevel?.basic?.conditions instanceof List ignore_failure: true @@ -261,6 +265,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_34d28d24 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -269,6 +274,7 @@ processors: tag: rename_asset_iamPolicy ignore_missing: true - foreach: + tag: foreach_json_asset_orgPolicy_e5a6ad79 field: json.asset.orgPolicy if: ctx.json?.asset?.orgPolicy instanceof List ignore_failure: true @@ -285,21 +291,23 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_asset_orgPolicy_bd402177 field: json.asset.orgPolicy if: ctx.json?.asset?.orgPolicy instanceof List ignore_failure: true processor: date: field: _ingest._value.updateTime - target_field: _ingest._value.updateTime + target_field: _ingest._value.updateTime tag: date_updateTime formats: - ISO8601 on_failure: - remove: - field: _ingest._value.updateTime + field: _ingest._value.updateTime ignore_missing: true - foreach: + tag: foreach_json_asset_orgPolicy_a4159288 field: json.asset.orgPolicy if: ctx.json?.asset?.orgPolicy instanceof List ignore_failure: true @@ -316,6 +324,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_asset_orgPolicy_e0d62d16 field: json.asset.orgPolicy if: ctx.json?.asset?.orgPolicy instanceof List ignore_failure: true @@ -414,6 +423,7 @@ processors: if: ctx.json?.asset?.osInventory?.updateTime != null && ctx.json.asset.osInventory.updateTime != '' on_failure: - append: + tag: append_error_message_4caa8fa0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -571,6 +581,7 @@ processors: if: ctx.json?.asset?.servicePerimeter?.spec?.vpcAccessibleServices?.enableRestriction != '' on_failure: - append: + tag: append_error_message_36b475e0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -592,6 +603,7 @@ processors: if: ctx.json?.asset?.servicePerimeter?.status?.vpcAccessibleServices?.enableRestriction != '' on_failure: - append: + tag: append_error_message_a4831c94 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -613,6 +625,7 @@ processors: if: ctx.json?.asset?.servicePerimeter?.useExplicitDryRunSpec != '' on_failure: - append: + tag: append_error_message_955fb45b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -855,6 +868,7 @@ processors: if: ctx.json?.asset?.updateTime != null && ctx.json.asset.updateTime != '' on_failure: - append: + tag: append_error_message_f27a63c5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: @@ -871,3 +885,7 @@ on_failure: - set: field: event.kind value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/google_scc/data_stream/asset/elasticsearch/ingest_pipeline/pipeline_prior_asset.yml b/packages/google_scc/data_stream/asset/elasticsearch/ingest_pipeline/pipeline_prior_asset.yml index 036ffd43a00..a89c176b5d4 100644 --- a/packages/google_scc/data_stream/asset/elasticsearch/ingest_pipeline/pipeline_prior_asset.yml +++ b/packages/google_scc/data_stream/asset/elasticsearch/ingest_pipeline/pipeline_prior_asset.yml @@ -56,6 +56,7 @@ processors: tag: rename_priorAsset_accessLevel_custom_expr ignore_missing: true - foreach: + tag: foreach_json_priorAsset_accessLevel_basic_conditions_2ff3ce4b field: json.priorAsset.accessLevel.basic.conditions if: ctx.json?.priorAsset?.accessLevel?.basic?.conditions instanceof List ignore_failure: true @@ -76,6 +77,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_priorAsset_accessLevel_basic_conditions_762cdc06 field: json.priorAsset.accessLevel.basic.conditions if: ctx.json?.priorAsset?.accessLevel?.basic?.conditions instanceof List ignore_failure: true @@ -92,6 +94,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_priorAsset_accessLevel_basic_conditions_91cf05ad field: json.priorAsset.accessLevel.basic.conditions if: ctx.json?.priorAsset?.accessLevel?.basic?.conditions instanceof List ignore_failure: true @@ -108,6 +111,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_priorAsset_accessLevel_basic_conditions_0931381d field: json.priorAsset.accessLevel.basic.conditions if: ctx.json?.priorAsset?.accessLevel?.basic?.conditions instanceof List ignore_failure: true @@ -261,6 +265,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_fc94d1d8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -269,6 +274,7 @@ processors: tag: rename_priorAsset_iamPolicy ignore_missing: true - foreach: + tag: foreach_json_priorAsset_orgPolicy_c13761ab field: json.priorAsset.orgPolicy if: ctx.json?.priorAsset?.orgPolicy instanceof List ignore_failure: true @@ -285,21 +291,23 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_priorAsset_orgPolicy_d569adbd field: json.priorAsset.orgPolicy if: ctx.json?.priorAsset?.orgPolicy instanceof List ignore_failure: true processor: date: field: _ingest._value.updateTime - target_field: _ingest._value.updateTime + target_field: _ingest._value.updateTime tag: date_updateTime formats: - ISO8601 on_failure: - remove: - field: _ingest._value.updateTime + field: _ingest._value.updateTime ignore_missing: true - foreach: + tag: foreach_json_priorAsset_orgPolicy_dba394d2 field: json.priorAsset.orgPolicy if: ctx.json?.priorAsset?.orgPolicy instanceof List ignore_failure: true @@ -316,6 +324,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_priorAsset_orgPolicy_5533a06c field: json.priorAsset.orgPolicy if: ctx.json?.priorAsset?.orgPolicy instanceof List ignore_failure: true @@ -414,6 +423,7 @@ processors: if: ctx.json?.priorAsset?.osInventory?.updateTime != null && ctx.json.priorAsset.osInventory.updateTime != '' on_failure: - append: + tag: append_error_message_247da014 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -571,6 +581,7 @@ processors: if: ctx.json?.priorAsset?.servicePerimeter?.spec?.vpcAccessibleServices?.enableRestriction != '' on_failure: - append: + tag: append_error_message_41382738 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -592,6 +603,7 @@ processors: if: ctx.json?.priorAsset?.servicePerimeter?.status?.vpcAccessibleServices?.enableRestriction != '' on_failure: - append: + tag: append_error_message_b8ef5338 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -613,6 +625,7 @@ processors: if: ctx.json?.priorAsset?.servicePerimeter?.useExplicitDryRunSpec != '' on_failure: - append: + tag: append_error_message_2773a0d7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -853,6 +866,7 @@ processors: if: ctx.json?.priorAsset?.updateTime != null && ctx.json.priorAsset.updateTime != '' on_failure: - append: + tag: append_error_message_79bb411d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: @@ -869,3 +883,7 @@ on_failure: - set: field: event.kind value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/google_scc/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/google_scc/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index e55de618e9a..41826453471 100644 --- a/packages/google_scc/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_scc/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -23,6 +23,7 @@ processors: ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: 'ctx.event?.original != null' @@ -37,6 +38,7 @@ processors: tag: json_event_original on_failure: - append: + tag: append_error_message_a6e0e953 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -48,6 +50,7 @@ processors: if: ctx.json?.timestamp != null && ctx.json.timestamp != '' on_failure: - append: + tag: append_error_message_16455195 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -58,6 +61,7 @@ processors: if: ctx.json?.timestamp != null && ctx.json.timestamp != '' on_failure: - append: + tag: append_error_message_697545f1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -79,6 +83,7 @@ processors: if: ctx.json?.protoPayload?.requestMetadata?.destinationAttributes?.ip != '' on_failure: - append: + tag: append_error_message_7e5f4057 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -101,6 +106,7 @@ processors: if: ctx.json?.protoPayload?.requestMetadata?.destinationAttributes?.port != '' on_failure: - append: + tag: append_error_message_07c58d16 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -137,6 +143,7 @@ processors: if: ctx.json?.protoPayload?.requestMetadata?.requestAttributes?.size != '' on_failure: - append: + tag: append_error_message_000e3aa2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -205,6 +212,7 @@ processors: ctx.google_scc.audit.severity.put('code',params.get(ctx.google_scc.audit.severity.value)); on_failure: - append: + tag: append_error_message_be1cfaca field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -221,6 +229,7 @@ processors: if: ctx.json?.sourceLocation?.line != '' on_failure: - append: + tag: append_error_message_b6d9b29e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -293,6 +302,7 @@ processors: tag: rename_request_metadata_caller_supplied_user_agent ignore_missing: true - user_agent: + tag: user_agent_google_scc_audit_proto_payload_request_metadata_caller_supplied_user_agent_a6dffcb2 field: google_scc.audit.proto_payload.request_metadata.caller.supplied_user_agent ignore_missing: true - convert: @@ -304,6 +314,7 @@ processors: if: ctx.json?.httpRequest?.cacheFillBytes != '' on_failure: - append: + tag: append_error_message_31789f79 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -315,6 +326,7 @@ processors: if: ctx.json?.httpRequest?.cacheHit != '' on_failure: - append: + tag: append_error_message_f97c1bd5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -326,6 +338,7 @@ processors: if: ctx.json?.httpRequest?.cacheLookup != '' on_failure: - append: + tag: append_error_message_29deea99 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -337,6 +350,7 @@ processors: if: ctx.json?.httpRequest?.cacheValidatedWithOriginServer != '' on_failure: - append: + tag: append_error_message_d167f2b7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -363,6 +377,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_152ba461 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -385,6 +400,7 @@ processors: if: ctx.json?.httpRequest?.requestSize != '' on_failure: - append: + tag: append_error_message_e6d97b3c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -401,6 +417,7 @@ processors: if: ctx.json?.httpRequest?.responseSize != '' on_failure: - append: + tag: append_error_message_9370f804 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - grok: @@ -412,6 +429,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5d0f9e1e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -429,6 +447,7 @@ processors: if: ctx.json?.httpRequest?.status != '' on_failure: - append: + tag: append_error_message_96ae8de4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -450,6 +469,7 @@ processors: if: ctx.json?.operation?.first != '' on_failure: - append: + tag: append_error_message_1d5c5866 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -466,6 +486,7 @@ processors: if: ctx.json?.operation?.last != '' on_failure: - append: + tag: append_error_message_392732ee field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -484,6 +505,7 @@ processors: tag: rename_authentication_info_principal_subject ignore_missing: true - foreach: + tag: foreach_json_protoPayload_authenticationInfo_serviceAccountDelegationInfo_4f0760aa field: json.protoPayload.authenticationInfo.serviceAccountDelegationInfo if: ctx.json?.protoPayload?.authenticationInfo?.serviceAccountDelegationInfo instanceof List ignore_failure: true @@ -494,6 +516,7 @@ processors: tag: rename_authentication_info_service_account_delegation_info_first_party_principal_email ignore_missing: true - foreach: + tag: foreach_json_protoPayload_authenticationInfo_serviceAccountDelegationInfo_7071bd42 field: json.protoPayload.authenticationInfo.serviceAccountDelegationInfo if: ctx.json?.protoPayload?.authenticationInfo?.serviceAccountDelegationInfo instanceof List ignore_failure: true @@ -504,6 +527,7 @@ processors: tag: rename_authentication_info_service_account_delegation_info_first_party_principal_service_metadata ignore_missing: true - foreach: + tag: foreach_json_protoPayload_authenticationInfo_serviceAccountDelegationInfo_de79d797 field: json.protoPayload.authenticationInfo.serviceAccountDelegationInfo if: ctx.json?.protoPayload?.authenticationInfo?.serviceAccountDelegationInfo instanceof List ignore_failure: true @@ -514,6 +538,7 @@ processors: tag: rename_authentication_info_service_account_delegation_info_principal_subject ignore_missing: true - foreach: + tag: foreach_json_protoPayload_authenticationInfo_serviceAccountDelegationInfo_be538c91 field: json.protoPayload.authenticationInfo.serviceAccountDelegationInfo if: ctx.json?.protoPayload?.authenticationInfo?.serviceAccountDelegationInfo instanceof List ignore_failure: true @@ -531,7 +556,7 @@ processors: - rename: field: json.protoPayload.authenticationInfo.serviceAccountKeyName target_field: google_scc.audit.proto_payload.authentication_info.service_account_key_name - tag: rename_authentication_info_service_account_key_name + tag: rename_json_protoPayload_authenticationInfo_serviceAccountKeyName_to_google_scc_audit_proto_payload_authentication_info_service_account_key_name_c8b37f59 ignore_missing: true - rename: field: json.protoPayload.authenticationInfo.thirdPartyPrincipal @@ -539,6 +564,7 @@ processors: tag: rename_authentication_info_third_party_principal ignore_missing: true - foreach: + tag: foreach_json_protoPayload_authorizationInfo_99757976 field: json.protoPayload.authorizationInfo if: ctx.json?.protoPayload?.authorizationInfo instanceof List ignore_failure: true @@ -556,6 +582,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_protoPayload_authorizationInfo_368b576c field: json.protoPayload.authorizationInfo if: ctx.json?.protoPayload?.authorizationInfo instanceof List ignore_failure: true @@ -571,6 +598,7 @@ processors: field: _ingest._value.resourceAttributes.createTime ignore_missing: true - foreach: + tag: foreach_json_protoPayload_authorizationInfo_a885d65e field: json.protoPayload.authorizationInfo if: ctx.json?.protoPayload?.authorizationInfo instanceof List ignore_failure: true @@ -586,6 +614,7 @@ processors: field: _ingest._value.resourceAttributes.deleteTime ignore_missing: true - foreach: + tag: foreach_json_protoPayload_authorizationInfo_c3e3bb22 field: json.protoPayload.authorizationInfo if: ctx.json?.protoPayload?.authorizationInfo instanceof List ignore_failure: true @@ -601,6 +630,7 @@ processors: field: _ingest._value.resourceAttributes.updateTime ignore_missing: true - foreach: + tag: foreach_json_protoPayload_authorizationInfo_43410c15 field: json.protoPayload.authorizationInfo if: ctx.json?.protoPayload?.authorizationInfo instanceof List ignore_failure: true @@ -611,6 +641,7 @@ processors: tag: rename_authorization_info_resource_attributes_display_name ignore_missing: true - foreach: + tag: foreach_json_protoPayload_authorizationInfo_e51aed96 field: json.protoPayload.authorizationInfo if: ctx.json?.protoPayload?.authorizationInfo instanceof List ignore_failure: true @@ -623,6 +654,7 @@ processors: ignore_missing: true tag: remove_authorization_info_resource_attributes_fields - foreach: + tag: foreach_json_protoPayload_authorizationInfo_b7c91650 field: json.protoPayload.authorizationInfo if: ctx.json?.protoPayload?.authorizationInfo instanceof List ignore_failure: true @@ -651,6 +683,7 @@ processors: if: ctx.json?.protoPayload?.numResponseItems != '' on_failure: - append: + tag: append_error_message_16961f54 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -782,6 +815,7 @@ processors: if: ctx.json?.protoPayload?.requestMetadata?.requestAttributes?.time != null && ctx.json.protoPayload.requestMetadata.requestAttributes.time != '' on_failure: - append: + tag: append_error_message_6df0af89 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -823,6 +857,7 @@ processors: if: ctx.json?.protoPayload?.status?.code != '' on_failure: - append: + tag: append_error_message_65d96254 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -849,6 +884,7 @@ processors: if: ctx.json?.receiveTimestamp != null && ctx.json.receiveTimestamp != '' on_failure: - append: + tag: append_error_message_7742568c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -875,6 +911,7 @@ processors: if: ctx.json?.split?.index != '' on_failure: - append: + tag: append_error_message_2be1a3fa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -886,6 +923,7 @@ processors: if: ctx.json?.split?.totalSplits != '' on_failure: - append: + tag: append_error_message_0b57e3da field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -907,6 +945,7 @@ processors: if: ctx.json?.traceSampled != '' on_failure: - append: + tag: append_error_message_47bdb898 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: @@ -961,6 +1000,7 @@ processors: tag: set_pipeline_error_into_event_kind if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/google_scc/data_stream/finding/elasticsearch/ingest_pipeline/default.yml b/packages/google_scc/data_stream/finding/elasticsearch/ingest_pipeline/default.yml index aa017327812..61fc74131e8 100644 --- a/packages/google_scc/data_stream/finding/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_scc/data_stream/finding/elasticsearch/ingest_pipeline/default.yml @@ -16,7 +16,7 @@ processors: field: ecs.version value: 8.11.0 tag: set_ecs_version - + # parse the event JSON - rename: field: message @@ -37,6 +37,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_8075efaa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - drop: @@ -128,8 +129,10 @@ processors: if: ctx.google_scc?.finding?.event_time != null && ctx.google_scc.finding.event_time != '' on_failure: - remove: + tag: remove_google_scc_finding_event_time_d4a60c67 field: google_scc.finding.event_time - append: + tag: append_error_message_fad3278a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -141,11 +144,14 @@ processors: if: ctx.google_scc?.finding?.create_time != null && ctx.google_scc.finding.create_time != '' on_failure: - remove: + tag: remove_google_scc_finding_create_time_1117d49a field: google_scc.finding.create_time - append: + tag: append_error_message_bc4a23db field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_google_scc_finding_connections_1b274adb field: google_scc.finding.connections if: ctx.google_scc?.finding?.connections instanceof List processor: @@ -162,6 +168,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_google_scc_finding_connections_c6bf161d field: google_scc.finding.connections if: ctx.google_scc?.finding?.connections instanceof List processor: @@ -178,6 +185,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_google_scc_finding_connections_4d765be3 field: google_scc.finding.connections if: ctx.google_scc?.finding?.connections instanceof List processor: @@ -194,6 +202,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_google_scc_finding_connections_d1e9fcfb field: google_scc.finding.connections if: ctx.google_scc?.finding?.connections instanceof List processor: @@ -210,6 +219,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_google_scc_finding_connections_42af5588 field: google_scc.finding.connections if: ctx.google_scc?.finding?.connections instanceof List processor: @@ -242,6 +252,7 @@ processors: tag: rename_finding_mitre_attack_primary_techniques ignore_missing: true - foreach: + tag: foreach_google_scc_finding_log_entries_62230acd field: google_scc.finding.log_entries if: ctx.google_scc?.finding?.log_entries instanceof List processor: @@ -263,8 +274,10 @@ processors: if: ctx.google_scc?.finding?.access?.caller_ip != '' on_failure: - remove: + tag: remove_google_scc_finding_access_caller_ip_f4606b68 field: google_scc.finding.access.caller_ip - append: + tag: append_error_message_245ddc14 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -278,6 +291,7 @@ processors: tag: rename_finding_access_principal_subject ignore_missing: true - foreach: + tag: foreach_google_scc_finding_access_service_account_delegation_info_7daf7025 field: google_scc.finding.access.service_account_delegation_info if: ctx.google_scc?.finding?.access?.service_account_delegation_info instanceof List processor: @@ -287,6 +301,7 @@ processors: tag: rename_finding_access_service_account_delegation_info_principal_email ignore_missing: true - foreach: + tag: foreach_google_scc_finding_access_service_account_delegation_info_94239a7d field: google_scc.finding.access.service_account_delegation_info if: ctx.google_scc?.finding?.access?.service_account_delegation_info instanceof List processor: @@ -324,6 +339,7 @@ processors: if: ctx.google_scc?.finding?.cloud_dlp_inspection?.full_scan != '' on_failure: - append: + tag: append_error_message_0c04f107 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -335,6 +351,7 @@ processors: if: ctx.google_scc?.finding?.cloud_dlp_inspection?.info_type_count != '' on_failure: - append: + tag: append_error_message_1ca51d7c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -396,6 +413,7 @@ processors: ignore_missing: true override: true - foreach: + tag: foreach_google_scc_finding_files_ef8b6a41 field: google_scc.finding.files if: ctx.google_scc?.finding?.files instanceof List processor: @@ -411,6 +429,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_google_scc_finding_files_65acaecd field: google_scc.finding.files if: ctx.google_scc?.finding?.files instanceof List processor: @@ -426,6 +445,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_google_scc_finding_files_437153fe field: google_scc.finding.files if: ctx.google_scc?.finding?.files instanceof List processor: @@ -441,6 +461,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_google_scc_finding_indicator_ip_addresses_7a614d28 field: google_scc.finding.indicator.ip_addresses if: ctx.google_scc?.finding?.indicator?.ip_addresses instanceof List processor: @@ -456,6 +477,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_google_scc_finding_indicator_signatures_b6f927c8 field: google_scc.finding.indicator.signatures if: ctx.google_scc?.finding?.indicator?.signatures instanceof List processor: @@ -475,6 +497,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}''Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_google_scc_finding_indicator_signatures_3dfcce68 field: google_scc.finding.indicator.signatures if: ctx.google_scc?.finding?.indicator?.signatures instanceof List processor: @@ -492,6 +515,7 @@ processors: if: ctx.google_scc?.finding?.kernel_root_kit?.unexpected_code_modification != '' on_failure: - append: + tag: append_error_message_776675ed field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -503,6 +527,7 @@ processors: if: ctx.google_scc?.finding?.kernel_root_kit?.unexpected_ftrace_handler != '' on_failure: - append: + tag: append_error_message_33f31e13 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -514,6 +539,7 @@ processors: if: ctx.google_scc?.finding?.kernel_root_kit?.unexpected_interrupt_handler != '' on_failure: - append: + tag: append_error_message_b607c991 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -525,6 +551,7 @@ processors: if: ctx.google_scc?.finding?.kernel_root_kit?.unexpected_kernel_code_pages != '' on_failure: - append: + tag: append_error_message_cde9ddad field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -536,6 +563,7 @@ processors: if: ctx.google_scc?.finding?.kernel_root_kit?.unexpected_kprobe_handler != '' on_failure: - append: + tag: append_error_message_689162a3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -547,6 +575,7 @@ processors: if: ctx.google_scc?.finding?.kernel_root_kit?.unexpected_processes_in_runqueue != '' on_failure: - append: + tag: append_error_message_0d7b43ed field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -558,6 +587,7 @@ processors: if: ctx.google_scc?.finding?.kernel_root_kit?.unexpected_read_only_data_modification != '' on_failure: - append: + tag: append_error_message_1c238b79 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -569,6 +599,7 @@ processors: if: ctx.google_scc?.finding?.kernel_root_kit?.unexpected_system_call_handler != '' on_failure: - append: + tag: append_error_message_aa88ff6d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -590,6 +621,7 @@ processors: if: ctx.google_scc?.finding?.mute_update_time != null && ctx.google_scc.finding.mute_update_time != '' on_failure: - append: + tag: append_error_message_b7ad7958 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -601,11 +633,14 @@ processors: if: ctx.google_scc?.finding?.mute_info?.static_mute?.apply_time != null && ctx.google_scc.finding.mute_info.static_mute.apply_time != '' on_failure: - remove: + tag: remove_google_scc_finding_mute_info_static_mute_apply_time_71d224c1 field: google_scc.finding.mute_info.static_mute.apply_time - append: + tag: append_error_message_c03d12b4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_google_scc_finding_mute_info_dynamic_mute_records_195618b3 field: google_scc.finding.mute_info.dynamic_mute_records if: ctx.google_scc?.finding?.mute_info?.dynamic_mute_records instanceof List processor: @@ -620,6 +655,7 @@ processors: field: _ingest._value.match_time ignore_missing: true - foreach: + tag: foreach_google_scc_finding_processes_9c5faff1 field: google_scc.finding.processes if: ctx.google_scc?.finding?.processes instanceof List processor: @@ -635,6 +671,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_google_scc_finding_processes_1a4860f3 field: google_scc.finding.processes if: ctx.google_scc?.finding?.processes instanceof List processor: @@ -650,6 +687,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_google_scc_finding_processes_f197957c field: google_scc.finding.processes if: ctx.google_scc?.finding?.processes instanceof List processor: @@ -665,6 +703,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_google_scc_finding_processes_0e491fec field: google_scc.finding.processes if: ctx.google_scc?.finding?.processes instanceof List processor: @@ -684,6 +723,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_google_scc_finding_processes_e2b97c12 field: google_scc.finding.processes if: ctx.google_scc?.finding?.processes instanceof List processor: @@ -703,6 +743,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_google_scc_finding_processes_b6537bbd field: google_scc.finding.processes if: ctx.google_scc?.finding?.processes instanceof List processor: @@ -722,6 +763,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_google_scc_finding_processes_bf4abadd field: google_scc.finding.processes if: ctx.google_scc?.finding?.processes instanceof List processor: @@ -737,6 +779,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_google_scc_finding_processes_3ff9c9ab field: google_scc.finding.processes if: ctx.google_scc?.finding?.processes instanceof List processor: @@ -752,6 +795,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_google_scc_finding_processes_44b17f2e field: google_scc.finding.processes if: ctx.google_scc?.finding?.processes instanceof List processor: @@ -767,6 +811,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_google_scc_finding_processes_390e738e field: google_scc.finding.processes if: ctx.google_scc?.finding?.processes instanceof List processor: @@ -826,8 +871,10 @@ processors: if: ctx.google_scc?.finding?.vulnerability?.cve?.cvssv3?.base_score != '' on_failure: - remove: + tag: remove_google_scc_finding_vulnerability_cve_cvssv3_base_score_99ddd8eb field: google_scc.finding.vulnerability.cve.cvssv3.base_score - append: + tag: append_error_message_a10a85c2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -839,8 +886,10 @@ processors: if: ctx.google_scc?.finding?.vulnerability?.cve?.exploit_release_date != null && ctx.google_scc.finding.vulnerability.cve.exploit_release_date != '' on_failure: - remove: + tag: remove_google_scc_finding_vulnerability_cve_exploit_release_date_124be961 field: google_scc.finding.vulnerability.cve.exploit_release_date - append: + tag: append_error_message_de217f00 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -852,8 +901,10 @@ processors: if: ctx.google_scc?.finding?.vulnerability?.cve?.first_exploitation_date != null && ctx.google_scc.finding.vulnerability.cve.first_exploitation_date != '' on_failure: - remove: + tag: remove_google_scc_finding_vulnerability_cve_first_exploitation_date_23450d73 field: google_scc.finding.vulnerability.cve.first_exploitation_date - append: + tag: append_error_message_4b7d9c1e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -864,8 +915,10 @@ processors: if: ctx.google_scc?.finding?.vulnerability?.cve?.observed_in_the_wild != '' on_failure: - remove: + tag: remove_google_scc_finding_vulnerability_cve_observed_in_the_wild_9f0cc880 field: google_scc.finding.vulnerability.cve.observed_in_the_wild - append: + tag: append_error_message_a74fd8ec field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -876,8 +929,10 @@ processors: if: ctx.google_scc?.finding?.vulnerability?.cve?.upstream_fix_available != '' on_failure: - remove: + tag: remove_google_scc_finding_vulnerability_cve_upstream_fix_available_f6f0465a field: google_scc.finding.vulnerability.cve.upstream_fix_available - append: + tag: append_error_message_df3c14fc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -888,8 +943,10 @@ processors: if: ctx.google_scc?.finding?.vulnerability?.cve?.zero_day != '' on_failure: - remove: + tag: remove_google_scc_finding_vulnerability_cve_zero_day_e4de89a0 field: google_scc.finding.vulnerability.cve.zero_day - append: + tag: append_error_message_d4e59adc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -910,8 +967,10 @@ processors: if: ctx.google_scc?.finding?.vulnerability?.provider_risk_score != '' on_failure: - remove: + tag: remove_google_scc_finding_vulnerability_provider_risk_score_1bb08f05 field: google_scc.finding.vulnerability.provider_risk_score - append: + tag: append_error_message_fe37b6a8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -922,8 +981,10 @@ processors: if: ctx.google_scc?.finding?.vulnerability?.reachable != '' on_failure: - remove: + tag: remove_google_scc_finding_vulnerability_reachable_8ff4d940 field: google_scc.finding.vulnerability.reachable - append: + tag: append_error_message_8b0673b0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -935,13 +996,15 @@ processors: if: ctx.google_scc?.finding?.vulnerability?.security_bulletin?.submission_time != null && ctx.google_scc.finding.vulnerability.security_bulletin.submission_time != '' on_failure: - remove: + tag: remove_google_scc_finding_vulnerability_security_bulletin_submission_time_938afefb field: google_scc.finding.vulnerability.security_bulletin.submission_time - append: + tag: append_error_message_60942d76 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - # populate ECS fields + # populate ECS fields - set: field: '@timestamp' tag: set_@timestamp_from_event_time @@ -1037,6 +1100,7 @@ processors: } on_failure: - append: + tag: append_error_message_36fdad6f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' @@ -1048,6 +1112,7 @@ processors: # container.* - foreach: + tag: foreach_google_scc_finding_containers_61a844ba field: google_scc.finding.containers if: ctx.google_scc?.finding?.containers instanceof List processor: @@ -1067,6 +1132,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_63714b6d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1111,6 +1177,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4e2a946d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1144,6 +1211,7 @@ processors: # destination.* - foreach: + tag: foreach_google_scc_finding_connections_f835726e field: google_scc.finding.connections if: ctx.google_scc?.finding?.connections instanceof List processor: @@ -1153,6 +1221,7 @@ processors: value: '{{{_ingest._value.destination.ip}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_connections_b3a171e6 field: google_scc.finding.connections if: ctx.google_scc?.finding?.connections instanceof List processor: @@ -1164,6 +1233,7 @@ processors: # We are not appending error.message when the convert processor fails because it generates an error related to an empty string. # An empty string is appended when google_scc.finding.connections[].destination.port is null in earlier processor. - foreach: + tag: foreach_destination_port_d457afd5 field: destination.port if: ctx.destination?.port instanceof List processor: @@ -1178,6 +1248,7 @@ processors: # source.* - foreach: + tag: foreach_google_scc_finding_connections_04d4aade field: google_scc.finding.connections if: ctx.google_scc?.finding?.connections instanceof List processor: @@ -1187,6 +1258,7 @@ processors: value: '{{{_ingest._value.source.ip}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_connections_172b74be field: google_scc.finding.connections if: ctx.google_scc?.finding?.connections instanceof List processor: @@ -1198,6 +1270,7 @@ processors: # We are not appending error.message when the convert processor fails because it generates an error related to an empty string. # An empty string is appended when google_scc.finding.connections[].source.port is null in earlier processor. - foreach: + tag: foreach_source_port_4be95c9c field: source.port if: ctx.source?.port instanceof List processor: @@ -1212,6 +1285,7 @@ processors: # network.* - foreach: + tag: foreach_google_scc_finding_connections_b249b289 field: google_scc.finding.connections if: ctx.google_scc?.finding?.connections instanceof List processor: @@ -1235,11 +1309,13 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_0240d94f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' # orchestrator.* - foreach: + tag: foreach_google_scc_finding_kubernetes_access_reviews_08520c7e field: google_scc.finding.kubernetes.access_reviews if: ctx.google_scc?.finding?.kubernetes?.access_reviews instanceof List processor: @@ -1249,6 +1325,7 @@ processors: value: '{{{_ingest._value.ns}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_kubernetes_access_reviews_50a1f89b field: google_scc.finding.kubernetes.access_reviews if: ctx.google_scc?.finding?.kubernetes?.access_reviews instanceof List processor: @@ -1258,6 +1335,7 @@ processors: value: '{{{_ingest._value.name}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_kubernetes_access_reviews_e0f50d28 field: google_scc.finding.kubernetes.access_reviews if: ctx.google_scc?.finding?.kubernetes?.access_reviews instanceof List processor: @@ -1267,6 +1345,7 @@ processors: value: '{{{_ingest._value.version}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_kubernetes_access_reviews_fef7fedd field: google_scc.finding.kubernetes.access_reviews if: ctx.google_scc?.finding?.kubernetes?.access_reviews instanceof List processor: @@ -1283,6 +1362,7 @@ processors: # file.* - foreach: + tag: foreach_google_scc_finding_files_672438eb field: google_scc.finding.files if: ctx.google_scc?.finding?.files instanceof List processor: @@ -1292,6 +1372,7 @@ processors: value: '{{{_ingest._value.path}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_files_3b016fef field: google_scc.finding.files if: ctx.google_scc?.finding?.files instanceof List processor: @@ -1303,6 +1384,7 @@ processors: # We are not appending error.message when the convert processor fails because it generates an error related to an empty string. # An empty string is appended when google_scc.finding.files[].size is null in earlier processor. - foreach: + tag: foreach_file_size_742f0b0f field: file.size if: ctx.file?.size instanceof List processor: @@ -1315,6 +1397,7 @@ processors: - remove: field: _ingest._value - foreach: + tag: foreach_google_scc_finding_files_b96466c4 field: google_scc.finding.files if: ctx.google_scc?.finding?.files instanceof List processor: @@ -1343,6 +1426,7 @@ processors: # process.* - foreach: + tag: foreach_google_scc_finding_processes_716f2d87 field: google_scc.finding.processes if: ctx.google_scc?.finding?.processes instanceof List processor: @@ -1354,6 +1438,7 @@ processors: # We are not appending error.message when the convert processor fails because it generates an error related to an empty string. # An empty string is appended when google_scc.finding.processes[].parent.pid is null in earlier processor. - foreach: + tag: foreach_process_parent_pid_07f44f04 field: process.parent.pid if: ctx.process?.parent?.pid instanceof List processor: @@ -1366,6 +1451,7 @@ processors: - remove: field: _ingest._value - foreach: + tag: foreach_google_scc_finding_processes_0d81c9eb field: google_scc.finding.processes if: ctx.google_scc?.finding?.processes instanceof List processor: @@ -1377,6 +1463,7 @@ processors: # We are not appending error.message when the convert processor fails because it generates an error related to an empty string. # An empty string is appended when google_scc.finding.processes[].pid is null in earlier processor. - foreach: + tag: foreach_process_pid_6bef8188 field: process.pid if: ctx.process?.pid instanceof List processor: @@ -1389,6 +1476,7 @@ processors: - remove: field: _ingest._value - foreach: + tag: foreach_google_scc_finding_processes_e375d47b field: google_scc.finding.processes if: ctx.google_scc?.finding?.processes instanceof List processor: @@ -1461,6 +1549,7 @@ processors: # threat.* - foreach: + tag: foreach_google_scc_finding_mitre_attack_additional_tactics_9c4e4b53 field: google_scc.finding.mitre_attack.additional.tactics if: ctx.google_scc?.finding?.mitre_attack?.additional?.tactics instanceof List processor: @@ -1476,6 +1565,7 @@ processors: allow_duplicates: false if: ctx.google_scc?.finding?.mitre_attack?.primary?.tactic != null - foreach: + tag: foreach_google_scc_finding_mitre_attack_additional_techniques_45a0ff97 field: google_scc.finding.mitre_attack.additional.techniques if: ctx.google_scc?.finding?.mitre_attack?.additional?.techniques instanceof List processor: @@ -1485,6 +1575,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_mitre_attack_primary_techniques_dff63b58 field: google_scc.finding.mitre_attack.primary.techniques if: ctx.google_scc?.finding?.mitre_attack?.primary?.techniques instanceof List processor: @@ -1674,6 +1765,7 @@ processors: } on_failure: - append: + tag: append_error_message_fd790b0a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' @@ -1690,6 +1782,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e9aa5c07 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' @@ -1711,6 +1804,7 @@ processors: copy_from: google_scc.finding.access.principal.email ignore_empty_value: true - foreach: + tag: foreach_google_scc_finding_access_service_account_delegation_info_c6c0960b field: google_scc.finding.access.service_account_delegation_info if: ctx.google_scc?.finding?.access?.service_account_delegation_info instanceof List && ctx.user?.id == null processor: @@ -1720,6 +1814,7 @@ processors: value: '{{{_ingest._value.principal.email}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_data_access_events_741213a8 field: google_scc.finding.data_access_events if: ctx.google_scc?.finding?.data_access_events instanceof List && ctx.user?.id == null processor: @@ -1735,6 +1830,7 @@ processors: ignore_empty_value: true if: ctx.user?.name == null - foreach: + tag: foreach_google_scc_finding_iam_bindings_37a7b9a6 field: google_scc.finding.iam_bindings if: ctx.google_scc?.finding?.iam_bindings instanceof List processor: @@ -1783,6 +1879,7 @@ processors: ignore_empty_value: true if: ctx.google_scc?.finding?.class == 'VULNERABILITY' - foreach: + tag: foreach_google_scc_finding_vulnerability_cve_references_54296b0e field: google_scc.finding.vulnerability.cve.references if: ctx.google_scc?.finding?.vulnerability?.cve?.references instanceof List processor: @@ -1821,6 +1918,7 @@ processors: } on_failure: - append: + tag: append_error_message_4885fd58 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1858,6 +1956,7 @@ processors: allow_duplicates: false if: ctx.google_scc?.finding?.resource?.name != null && ctx.google_scc.finding.resource.type == 'google.compute.Instance' - foreach: + tag: foreach_google_scc_finding_connections_730bcc96 field: google_scc.finding.connections if: ctx.google_scc?.finding?.connections instanceof List processor: @@ -1867,6 +1966,7 @@ processors: value: '{{{_ingest._value.destination.ip}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_connections_f82a1f44 field: google_scc.finding.connections if: ctx.google_scc?.finding?.connections instanceof List processor: @@ -1876,6 +1976,7 @@ processors: value: '{{{_ingest._value.source.ip}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_indicator_ip_addresses_3f303dc0 field: google_scc.finding.indicator.ip_addresses if: ctx.google_scc?.finding?.indicator?.ip_addresses instanceof List processor: @@ -1909,6 +2010,7 @@ processors: allow_duplicates: false if: ctx.google_scc?.finding?.access?.principal?.email != null - foreach: + tag: foreach_google_scc_finding_access_service_account_delegation_info_85850161 field: google_scc.finding.access.service_account_delegation_info if: ctx.google_scc?.finding?.access?.service_account_delegation_info instanceof List processor: @@ -1918,6 +2020,7 @@ processors: value: '{{{_ingest._value.principal.email}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_data_access_events_a9857acc field: google_scc.finding.data_access_events if: ctx.google_scc?.finding?.data_access_events instanceof List processor: @@ -1927,6 +2030,7 @@ processors: value: '{{{_ingest._value.principal_email}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_contacts_billing_d68bc0d1 field: google_scc.finding.contacts.billing if: ctx.google_scc?.finding?.contacts?.billing instanceof List processor: @@ -1936,6 +2040,7 @@ processors: value: '{{{_ingest._value.email}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_contacts_legal_30a9452f field: google_scc.finding.contacts.legal if: ctx.google_scc?.finding?.contacts?.legal instanceof List processor: @@ -1945,6 +2050,7 @@ processors: value: '{{{_ingest._value.email}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_contacts_security_30e22294 field: google_scc.finding.contacts.security if: ctx.google_scc?.finding?.contacts?.security instanceof List processor: @@ -1954,6 +2060,7 @@ processors: value: '{{{_ingest._value.email}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_contacts_all_1d3e987d field: google_scc.finding.contacts.all if: ctx.google_scc?.finding?.contacts?.all instanceof List processor: @@ -1963,6 +2070,7 @@ processors: value: '{{{_ingest._value.email}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_contacts_product_updates_3956a39c field: google_scc.finding.contacts.product_updates if: ctx.google_scc?.finding?.contacts?.product_updates instanceof List processor: @@ -1972,6 +2080,7 @@ processors: value: '{{{_ingest._value.email}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_contacts_suspension_9b47e711 field: google_scc.finding.contacts.suspension if: ctx.google_scc?.finding?.contacts?.suspension instanceof List processor: @@ -1981,6 +2090,7 @@ processors: value: '{{{_ingest._value.email}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_contacts_technical_10ffeeb5 field: google_scc.finding.contacts.technical if: ctx.google_scc?.finding?.contacts?.technical instanceof List processor: @@ -1990,6 +2100,7 @@ processors: value: '{{{_ingest._value.email}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_contacts_technical_incidents_a1247a35 field: google_scc.finding.contacts.technical_incidents if: ctx.google_scc?.finding?.contacts?.technical_incidents instanceof List processor: @@ -1999,6 +2110,7 @@ processors: value: '{{{_ingest._value.email}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_processes_d71833ca field: google_scc.finding.processes if: ctx.google_scc?.finding?.processes instanceof List processor: @@ -2008,6 +2120,7 @@ processors: value: '{{{_ingest._value.script.sha256}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_processes_1bf5e6e6 field: google_scc.finding.processes if: ctx.google_scc?.finding?.processes instanceof List processor: @@ -2017,6 +2130,7 @@ processors: value: '{{{_ingest._value.binary.sha256}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_processes_2d054eaa field: google_scc.finding.processes if: ctx.google_scc?.finding?.processes instanceof List processor: @@ -2030,6 +2144,7 @@ processors: value: '{{{_ingest._value.sha256}}}' allow_duplicates: false - foreach: + tag: foreach_google_scc_finding_files_a8f800c7 field: google_scc.finding.files if: ctx.google_scc?.finding?.files instanceof List processor: @@ -2057,6 +2172,7 @@ processors: tag: remove_unwanted_fields ignore_missing: true - foreach: + tag: foreach_google_scc_finding_vulnerability_cve_references_4141d886 field: google_scc.finding.vulnerability.cve.references if: ctx.google_scc?.finding?.vulnerability?.cve?.references instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) processor: @@ -2066,6 +2182,7 @@ processors: tag: remove_remove_duplicate_custom_fields_from_vulnerability_cve_references_array ignore_missing: true - foreach: + tag: foreach_google_scc_finding_connections_910090a9 field: google_scc.finding.connections if: ctx.google_scc?.finding?.connections instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) processor: @@ -2079,6 +2196,7 @@ processors: tag: remove_remove_duplicate_custom_fields_from_connections_array ignore_missing: true - foreach: + tag: foreach_google_scc_finding_containers_f6afa16b field: google_scc.finding.containers if: ctx.google_scc?.finding?.containers instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) processor: @@ -2088,6 +2206,7 @@ processors: tag: remove_remove_duplicate_custom_fields_from_containers_array ignore_missing: true - foreach: + tag: foreach_google_scc_finding_files_d922432e field: google_scc.finding.files if: ctx.google_scc?.finding?.files instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) processor: @@ -2099,6 +2218,7 @@ processors: tag: remove_remove_duplicate_custom_fields_from_files_array ignore_missing: true - foreach: + tag: foreach_google_scc_finding_processes_2e294952 field: google_scc.finding.processes if: ctx.google_scc?.finding?.processes instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) processor: @@ -2110,6 +2230,7 @@ processors: tag: remove_remove_duplicate_custom_fields_from_processes_array ignore_missing: true - foreach: + tag: foreach_google_scc_finding_iam_bindings_79a23a2c field: google_scc.finding.iam_bindings if: ctx.google_scc?.finding?.iam_bindings instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) processor: @@ -2119,6 +2240,7 @@ processors: tag: remove_remove_duplicate_custom_fields_from_iam_bindings_array ignore_missing: true - foreach: + tag: foreach_google_scc_finding_kubernetes_access_reviews_c6a3b8f9 field: google_scc.finding.kubernetes.access_reviews if: ctx.google_scc?.finding?.kubernetes?.access_reviews instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) processor: @@ -2190,6 +2312,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/google_scc/data_stream/source/elasticsearch/ingest_pipeline/default.yml b/packages/google_scc/data_stream/source/elasticsearch/ingest_pipeline/default.yml index b0c0a47d1f7..86b0d1f7303 100644 --- a/packages/google_scc/data_stream/source/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_scc/data_stream/source/elasticsearch/ingest_pipeline/default.yml @@ -23,6 +23,7 @@ processors: ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: 'ctx.event?.original != null' @@ -37,6 +38,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_8075efaa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - drop: @@ -76,6 +78,7 @@ processors: - '^organizations/%{DATA:organization.id}/sources/%{DATA:google_scc.source.id}$' on_failure: - append: + tag: append_error_message_75163151 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: @@ -112,6 +115,7 @@ processors: tag: set_pipeline_error_into_event_kind if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/google_scc/manifest.yml b/packages/google_scc/manifest.yml index ffd55af19b3..fae2d18b289 100644 --- a/packages/google_scc/manifest.yml +++ b/packages/google_scc/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.2.3" name: google_scc title: Google Security Command Center -version: "2.6.1" +version: "2.6.2" description: Collect logs from Google Security Command Center with Elastic Agent. type: integration categories: diff --git a/packages/google_secops/changelog.yml b/packages/google_secops/changelog.yml index 9df4cbebc36..4d400cb6596 100644 --- a/packages/google_secops/changelog.yml +++ b/packages/google_secops/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.3.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.3.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/google_secops/data_stream/alert/elasticsearch/ingest_pipeline/default.yml b/packages/google_secops/data_stream/alert/elasticsearch/ingest_pipeline/default.yml index 13cdb9a893b..f6d3121b0a8 100644 --- a/packages/google_secops/data_stream/alert/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_secops/data_stream/alert/elasticsearch/ingest_pipeline/default.yml @@ -39,6 +39,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -59,9 +60,11 @@ processors: if: ctx.google_secops?.alert?.createdTime != null && ctx.google_secops.alert.createdTime != '' on_failure: - remove: + tag: remove_google_secops_alert_createdTime_7dd1eb8f field: google_secops.alert.createdTime ignore_missing: true - append: + tag: append_error_message_35df434e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -81,9 +84,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_google_secops_alert_detection_riskScore_d48d1c3a field: google_secops.alert.detection.riskScore ignore_missing: true - append: + tag: append_error_message_26e89610 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -99,9 +104,11 @@ processors: if: ctx.google_secops?.alert?.detection?.variables?.risk_score?.int64Val != '' on_failure: - remove: + tag: remove_google_secops_alert_detection_variables_risk_score_int64Val_e278a26c field: google_secops.alert.detection.variables.risk_score.int64Val ignore_missing: true - append: + tag: append_error_message_051114d8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -112,9 +119,11 @@ processors: if: ctx.google_secops?.alert?.detection?.variables?.risk_score?.value != '' on_failure: - remove: + tag: remove_google_secops_alert_detection_variables_risk_score_value_e64c56eb field: google_secops.alert.detection.variables.risk_score.value ignore_missing: true - append: + tag: append_error_message_d905bf6e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -129,9 +138,9 @@ processors: ignore_empty_value: true - set: field: google_secops.alert.friendly_name - tag: set_rule_name_from_alert_detection_ruleName + tag: set_google_secops_alert_friendly_name_e7bfafb7 copy_from: google_secops.alert.detection.ruleName - ignore_empty_value: true + ignore_empty_value: true - set: field: rule.version tag: set_rule_version_from_alert_detection_ruleVersion @@ -146,9 +155,11 @@ processors: if: ctx.google_secops?.alert?.detectionTime != null && ctx.google_secops.alert.detectionTime != '' on_failure: - remove: + tag: remove_google_secops_alert_detectionTime_f86b9cba field: google_secops.alert.detectionTime ignore_missing: true - append: + tag: append_error_message_cec29068 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -163,9 +174,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_google_secops_alert_event_metadata_baseLabels_allowScopedAccess_a0a9056c field: google_secops.alert.event.metadata.baseLabels.allowScopedAccess ignore_missing: true - append: + tag: append_error_message_77f01024 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -180,9 +193,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_google_secops_alert_event_metadata_enrichmentLabels_allowScopedAccess_ca8dbb18 field: google_secops.alert.event.metadata.enrichmentLabels.allowScopedAccess ignore_missing: true - append: + tag: append_error_message_b3f9e92c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -194,9 +209,11 @@ processors: if: ctx.google_secops?.alert?.event?.metadata?.eventTimestamp != null && ctx.google_secops.alert.event.metadata.eventTimestamp != '' on_failure: - remove: + tag: remove_google_secops_alert_event_metadata_eventTimestamp_d2b37e05 field: google_secops.alert.event.metadata.eventTimestamp ignore_missing: true - append: + tag: append_error_message_c4dc74c2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -208,9 +225,11 @@ processors: if: ctx.google_secops?.alert?.event?.metadata?.ingestedTimestamp != null && ctx.google_secops.alert.event.metadata.ingestedTimestamp != '' on_failure: - remove: + tag: remove_google_secops_alert_event_metadata_ingestedTimestamp_ec3f600a field: google_secops.alert.event.metadata.ingestedTimestamp ignore_missing: true - append: + tag: append_error_message_e68d432c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -346,9 +365,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_google_secops_alert_event_network_http_responseCode_edb0d9bc field: google_secops.alert.event.network.http.responseCode ignore_missing: true - append: + tag: append_error_message_d3fad552 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -368,6 +389,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d82c4066 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -641,6 +663,7 @@ processors: ctx.host.geo.location = locationList; on_failure: - append: + tag: append_error_message_6b2c8e31 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -663,7 +686,7 @@ processors: ignore_missing: true - foreach: field: google_secops.alert.event.principal.ip - tag: foreach_append_event_principal_ip_into_host_ip + tag: foreach_append_event_principal_ip_into_host_ip if: ctx.google_secops?.alert?.event?.principal?.ip instanceof List processor: append: @@ -703,6 +726,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c4e630a3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - uppercase: @@ -711,6 +735,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f58efe9c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -720,9 +745,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_google_secops_alert_event_principal_port_2e9773a9 field: google_secops.alert.event.principal.port ignore_missing: true - append: + tag: append_error_message_c0992244 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -819,9 +846,11 @@ processors: if: ctx.google_secops?.alert?.event?.principal?.process?.parentProcess?.pid != '' on_failure: - remove: + tag: remove_google_secops_alert_event_principal_process_parentProcess_pid_462dfa31 field: google_secops.alert.event.principal.process.parentProcess.pid ignore_missing: true - append: + tag: append_error_message_9f1ea442 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -891,9 +920,11 @@ processors: if: ctx.google_secops?.alert?.event?.principal?.process?.pid != '' on_failure: - remove: + tag: remove_google_secops_alert_event_principal_process_pid_fb377268 field: google_secops.alert.event.principal.process.pid ignore_missing: true - append: + tag: append_error_message_bc84b39c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1033,6 +1064,7 @@ processors: } on_failure: - append: + tag: append_error_message_0527d0e8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1159,6 +1191,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_aed5673a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - uppercase: @@ -1167,6 +1200,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4c748822 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1245,9 +1279,11 @@ processors: if: ctx.google_secops?.alert?.event?.target?.file?.lastModificationTime != null && ctx.google_secops.alert.event.target.file.lastModificationTime != '' on_failure: - remove: + tag: remove_google_secops_alert_event_target_file_lastModificationTime_cd59c52c field: google_secops.alert.event.target.file.lastModificationTime ignore_missing: true - append: + tag: append_error_message_ade82b18 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -1339,6 +1375,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_aba1517d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - uppercase: @@ -1347,6 +1384,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_94c8b856 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1356,9 +1394,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_google_secops_alert_event_target_port_db037117 field: google_secops.alert.event.target.port ignore_missing: true - append: + tag: append_error_message_22698bd3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1375,9 +1415,11 @@ processors: if: ctx.google_secops?.alert?.event?.target?.process?.file?.firstSeenTime != null && ctx.google_secops.alert.event.target.process.file.firstSeenTime != '' on_failure: - remove: + tag: remove_google_secops_alert_event_target_process_file_firstSeenTime_2d722f10 field: google_secops.alert.event.target.process.file.firstSeenTime ignore_missing: true - append: + tag: append_error_message_36e1e382 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -1389,9 +1431,11 @@ processors: if: ctx.google_secops?.alert?.event?.target?.process?.file?.lastModificationTime != null && ctx.google_secops.alert.event.target.process.file.lastModificationTime != '' on_failure: - remove: + tag: remove_google_secops_alert_event_target_process_file_lastModificationTime_154bb379 field: google_secops.alert.event.target.process.file.lastModificationTime ignore_missing: true - append: + tag: append_error_message_e389018e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1496,9 +1540,11 @@ processors: if: ctx.google_secops?.alert?.timeWindow?.endTime != null && ctx.google_secops.alert.timeWindow.endTime != '' on_failure: - remove: + tag: remove_google_secops_alert_timeWindow_endTime_0459de6f field: google_secops.alert.timeWindow.endTime ignore_missing: true - append: + tag: append_error_message_63a2dc8a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1515,9 +1561,11 @@ processors: if: ctx.google_secops?.alert?.timeWindow?.startTime != null && ctx.google_secops.alert.timeWindow.startTime != '' on_failure: - remove: + tag: remove_google_secops_alert_timeWindow_startTime_10f41d3e field: google_secops.alert.timeWindow.startTime ignore_missing: true - append: + tag: append_error_message_adb919c0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1806,3 +1854,7 @@ on_failure: field: event.kind tag: set_pipeline_error_to_event_kind value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/google_secops/manifest.yml b/packages/google_secops/manifest.yml index e2fd1a874e9..21b9da1112a 100644 --- a/packages/google_secops/manifest.yml +++ b/packages/google_secops/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: google_secops title: Google SecOps -version: 1.3.1 +version: 1.3.2 description: Collect alerts from Google SecOps with Elastic Agent. type: integration categories: diff --git a/packages/google_workspace/changelog.yml b/packages/google_workspace/changelog.yml index c0ac96f32cb..335ff8a5f27 100644 --- a/packages/google_workspace/changelog.yml +++ b/packages/google_workspace/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.5.3" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "3.5.2" changes: - description: Add missing ECS event categorization for the gcp, rules, calendar, group_enterprise, context_aware_access, meet, chat, chrome, login, device, keep, and vault data streams. diff --git a/packages/google_workspace/data_stream/access_transparency/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/access_transparency/elasticsearch/ingest_pipeline/default.yml index 06441df852d..4dadf006d15 100644 --- a/packages/google_workspace/data_stream/access_transparency/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/access_transparency/elasticsearch/ingest_pipeline/default.yml @@ -13,23 +13,28 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - set: + tag: set_ecs_version_135371bc field: ecs.version value: '8.16.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_90c6475f field: event.original target_field: json on_failure: - append: + tag: append_error_message_45b808be field: error.message value: '{{{_ingest.on_failure_message}}}' - drop: @@ -37,9 +42,11 @@ processors: description: Discard events that are missing the target during the split operation and are subsequently returned as the root object. tag: drop_empty_events - set: + tag: set_event_kind_3a8924a2 field: event.kind value: [event] - date: + tag: date_json_id_time_9350ecd5 field: json.id.time if: ctx.json?.id?.time != null && ctx.json.id.time != '' formats: @@ -51,13 +58,16 @@ processors: - yyyy/MM/dd HH:mm:ss z on_failure: - append: + tag: append_error_message_18a82a92 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_time_01ac69ca field: google_workspace.id.time copy_from: '@timestamp' ignore_empty_value: true - fingerprint: + tag: fingerprint_52623c07 description: Hashes the ID object and uses it as the document id to avoid duplicate events. fields: - json.id @@ -65,73 +75,90 @@ processors: target_field: _id ignore_missing: true - rename: + tag: rename_json_events_name_to_event_action_75ca640e field: json.events.name target_field: event.action ignore_missing: true - set: + tag: set_google_workspace_event_name_69e4d7fd field: google_workspace.event.name copy_from: event.action ignore_empty_value: true - rename: + tag: rename_json_id_applicationName_to_event_provider_fffca35d field: json.id.applicationName target_field: event.provider ignore_missing: true - set: + tag: set_google_workspace_id_application_name_8030da4a field: google_workspace.id.application_name copy_from: event.provider ignore_empty_value: true - convert: + tag: convert_json_id_uniqueQualifier_to_event_id_0239362e field: json.id.uniqueQualifier target_field: event.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_ab6a6ac3 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_unique_qualifier_8c91fe06 field: google_workspace.id.unique_qualifier copy_from: event.id ignore_empty_value: true - rename: + tag: rename_json_actor_email_to_source_user_email_4ee5e9ab field: json.actor.email target_field: source.user.email ignore_missing: true - set: + tag: set_google_workspace_actor_email_9cd77425 field: google_workspace.actor.email copy_from: source.user.email ignore_empty_value: true - set: + tag: set_user_email_371886fa field: user.email copy_from: source.user.email ignore_empty_value: true - convert: + tag: convert_json_actor_profileId_to_source_user_id_e10534f1 field: json.actor.profileId target_field: source.user.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_dc10d8d6 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_actor_profile_id_6b7a4346 field: google_workspace.actor.profile.id copy_from: source.user.id ignore_empty_value: true - convert: + tag: convert_json_ipAddress_to_source_ip_6cbf48a8 field: json.ipAddress target_field: source.ip type: ip ignore_missing: true on_failure: - append: + tag: append_error_message_79a0c21d field: error.message value: '{{{_ingest.on_failure_message}}}' - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -140,59 +167,73 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - set: + tag: set_google_workspace_ip_address_b61dbbba field: google_workspace.ip_address copy_from: source.ip ignore_empty_value: true - rename: + tag: rename_json_kind_to_google_workspace_kind_8c6d9f29 field: json.kind target_field: google_workspace.kind ignore_missing: true - rename: + tag: rename_json_etag_to_google_workspace_etag_4bd4a803 field: json.etag target_field: google_workspace.etag ignore_missing: true - convert: + tag: convert_json_id_customerId_to_organization_id_ffd90a33 field: json.id.customerId target_field: organization.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_5aacea86 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_customer_id_46a3415a field: google_workspace.id.customer.id copy_from: organization.id ignore_empty_value: true - rename: + tag: rename_json_actor_callerType_to_google_workspace_actor_type_308eeffc field: json.actor.callerType target_field: google_workspace.actor.type ignore_missing: true - rename: + tag: rename_json_actor_key_to_google_workspace_actor_key_3f392d49 field: json.actor.key target_field: google_workspace.actor.key ignore_missing: true - rename: + tag: rename_json_ownerDomain_to_google_workspace_organization_domain_6580cd7d field: json.ownerDomain target_field: google_workspace.organization.domain ignore_missing: true - rename: + tag: rename_json_events_type_to_google_workspace_event_type_e73f1e18 field: json.events.type target_field: google_workspace.event.type ignore_missing: true - set: + tag: set_user_id_f89889ac field: user.id copy_from: source.user.id ignore_empty_value: true - script: + tag: script_a3a2cdbf lang: painless description: Extract User name and User domain from Email. if: ctx.source?.user?.email != null && ctx.source.user.email.contains("@") @@ -209,36 +250,43 @@ processors: ctx.user.domain = splitmail[1]; ctx.source.user.domain = splitmail[1]; - append: + tag: append_related_hosts_23e97a1f field: related.hosts value: '{{{user.domain}}}' if: ctx.user?.domain != null allow_duplicates: false - append: + tag: append_related_hosts_ea7c8eda field: related.hosts value: '{{{google_workspace.organization.domain}}}' if: ctx.google_workspace?.organization?.domain != null allow_duplicates: false - append: + tag: append_related_ip_8121c591 field: related.ip value: '{{{source.ip}}}' if: ctx.source?.ip != null allow_duplicates: false - append: + tag: append_related_user_bcb5a560 field: related.user value: '{{{source.user.id}}}' if: ctx.source?.user?.id != null allow_duplicates: false - append: + tag: append_related_user_f745a124 field: related.user value: '{{{source.user.name}}}' if: ctx.source?.user?.name != null allow_duplicates: false - append: + tag: append_related_user_8c49e24b field: related.user value: '{{{source.user.email}}}' if: ctx.source?.user?.name != null allow_duplicates: false - script: + tag: script_3afc7189 lang: painless description: Map Event parameters into key value pair. if: ctx.json?.events?.parameters != null && ctx.json.events.parameters instanceof List @@ -310,59 +358,73 @@ processors: ctx.json.events.parameters = parameters; } - rename: + tag: rename_google_workspace_access_transparency_ACCESS_APPROVAL_ALERT_CENTER_IDS_to_google_workspace_access_transparency_access_approval_alert_center_ids_d464e967 field: google_workspace.access_transparency.ACCESS_APPROVAL_ALERT_CENTER_IDS target_field: google_workspace.access_transparency.access_approval.alert_center_ids ignore_missing: true - rename: + tag: rename_google_workspace_access_transparency_ACCESS_APPROVAL_REQUEST_IDS_to_google_workspace_access_transparency_access_approval_request_ids_28bacd7b field: google_workspace.access_transparency.ACCESS_APPROVAL_REQUEST_IDS target_field: google_workspace.access_transparency.access_approval.request_ids ignore_missing: true - rename: + tag: rename_google_workspace_access_transparency_ACCESS_MANAGEMENT_POLICY_to_google_workspace_access_transparency_access_management_policy_3bf44065 field: google_workspace.access_transparency.ACCESS_MANAGEMENT_POLICY target_field: google_workspace.access_transparency.access_management.policy ignore_missing: true - rename: + tag: rename_google_workspace_access_transparency_ACTOR_HOME_OFFICE_to_google_workspace_access_transparency_actor_home_office_e31dc548 field: google_workspace.access_transparency.ACTOR_HOME_OFFICE target_field: google_workspace.access_transparency.actor_home_office ignore_missing: true - rename: + tag: rename_google_workspace_access_transparency_GSUITE_PRODUCT_NAME_to_google_workspace_access_transparency_gsuite_product_name_6da77240 field: google_workspace.access_transparency.GSUITE_PRODUCT_NAME target_field: google_workspace.access_transparency.gsuite_product_name ignore_missing: true - rename: + tag: rename_google_workspace_access_transparency_JUSTIFICATIONS_to_google_workspace_access_transparency_justifications_1ba4c224 field: google_workspace.access_transparency.JUSTIFICATIONS target_field: google_workspace.access_transparency.justifications ignore_missing: true - rename: + tag: rename_google_workspace_access_transparency_LOG_ID_to_google_workspace_access_transparency_log_id_e2cd02d6 field: google_workspace.access_transparency.LOG_ID target_field: google_workspace.access_transparency.log_id ignore_missing: true - rename: + tag: rename_google_workspace_access_transparency_ON_BEHALF_OF_to_google_workspace_access_transparency_on_behalf_of_3c1a916e field: google_workspace.access_transparency.ON_BEHALF_OF target_field: google_workspace.access_transparency.on_behalf_of ignore_missing: true - rename: + tag: rename_google_workspace_access_transparency_OWNER_EMAIL_to_google_workspace_access_transparency_owner_email_86017da4 field: google_workspace.access_transparency.OWNER_EMAIL target_field: google_workspace.access_transparency.owner_email ignore_missing: true - append: + tag: append_related_user_a146e0da field: related.user value: '{{{google_workspace.access_transparency.owner_email}}}' if: ctx.google_workspace?.access_transparency?.owner_email != null allow_duplicates: false - rename: + tag: rename_google_workspace_access_transparency_RESOURCE_NAME_to_google_workspace_access_transparency_resource_name_2c76c784 field: google_workspace.access_transparency.RESOURCE_NAME target_field: google_workspace.access_transparency.resource_name ignore_missing: true - rename: + tag: rename_google_workspace_access_transparency_TICKETS_to_google_workspace_access_transparency_tickets_67375b84 field: google_workspace.access_transparency.TICKETS target_field: google_workspace.access_transparency.tickets ignore_missing: true - remove: + tag: remove_e5a80934 field: - json ignore_missing: true - remove: + tag: remove_2a1ed52f if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - google_workspace.ip_address @@ -375,6 +437,7 @@ processors: - google_workspace.id.time ignore_missing: true - script: + tag: script_e2ccbdd7 description: Drops null/empty values recursively. lang: painless source: | @@ -392,10 +455,12 @@ processors: } dropEmptyFields(ctx); - set: + tag: set_event_kind_92954dfa field: event.kind value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/google_workspace/data_stream/admin/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/admin/elasticsearch/ingest_pipeline/default.yml index 49ea4c6cca3..ad2f8163afd 100644 --- a/packages/google_workspace/data_stream/admin/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/admin/elasticsearch/ingest_pipeline/default.yml @@ -13,22 +13,27 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - set: + tag: set_ecs_version_135371bc field: ecs.version value: '8.16.0' - append: + tag: append_event_category_576de059 field: event.category value: iam - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - drop: @@ -36,9 +41,11 @@ processors: description: Discard events that are missing the target during the split operation and are subsequently returned as the root object. tag: drop_empty_events - set: + tag: set_event_kind_de80643c field: event.kind value: event - date: + tag: date_json_id_time_8c30453c field: json.id.time if: ctx.json?.id?.time != null && ctx.json.id.time != '' timezone: UTC @@ -51,9 +58,11 @@ processors: - yyyy/MM/dd HH:mm z on_failure: - append: + tag: append_error_message_36333fb5 field: error.message value: "{{{_ingest.on_failure_message}}}" - fingerprint: + tag: fingerprint_567b61c4 description: Hashes the ID object and uses it as the document id to avoid duplicate events. fields: - json.id @@ -61,582 +70,121 @@ processors: target_field: _id ignore_missing: true ignore_failure: true - - rename: + - rename: + tag: rename_json_events_name_to_event_action_75ca640e field: json.events.name target_field: event.action ignore_missing: true - - rename: + - rename: + tag: rename_json_id_applicationName_to_event_provider_fffca35d field: json.id.applicationName target_field: event.provider ignore_missing: true - - rename: + - rename: + tag: rename_json_id_uniqueQualifier_to_event_id_2ab4dc15 field: json.id.uniqueQualifier target_field: event.id ignore_missing: true - - rename: + - rename: + tag: rename_json_actor_email_to_source_user_email_4ee5e9ab field: json.actor.email target_field: source.user.email ignore_missing: true - - set: + - set: + tag: set_user_email_f8429233 field: user.email copy_from: source.user.email ignore_failure: true - - rename: + - rename: + tag: rename_json_actor_profileId_to_source_user_id_d7a6e456 field: json.actor.profileId target_field: source.user.id ignore_missing: true - convert: + tag: convert_json_ipAddress_to_source_ip_56645655 field: json.ipAddress target_field: source.ip type: ip ignore_failure: true - - rename: + - rename: + tag: rename_json_kind_to_google_workspace_kind_8c6d9f29 field: json.kind target_field: google_workspace.kind ignore_missing: true - - rename: + - rename: + tag: rename_json_id_customerId_to_organization_id_7ada5886 field: json.id.customerId target_field: organization.id ignore_missing: true - - rename: + - rename: + tag: rename_json_actor_callerType_to_google_workspace_actor_type_308eeffc field: json.actor.callerType target_field: google_workspace.actor.type ignore_missing: true - - rename: + - rename: + tag: rename_json_actor_key_to_google_workspace_actor_key_3f392d49 field: json.actor.key target_field: google_workspace.actor.key ignore_missing: true - - rename: + - rename: + tag: rename_json_ownerDomain_to_google_workspace_organization_domain_6580cd7d field: json.ownerDomain target_field: google_workspace.organization.domain ignore_missing: true - - rename: + - rename: + tag: rename_json_events_type_to_google_workspace_event_type_e73f1e18 field: json.events.type target_field: google_workspace.event.type ignore_missing: true - - set: + - set: + tag: set_user_id_64f70b73 field: user.id copy_from: source.user.id if: ctx?.source?.user?.id != null - - append: + - append: + tag: append_event_category_88c0ad94 field: event.category value: configuration - if: '[ - "ADMIN_EVENTS_TOGGLE_NEW_APP_FEATURES_PREFERENCE", - "ALLOW_SERVICE_FOR_OAUTH2_ACCESS", - "ALLOW_STRONG_AUTHENTICATION", - "CHANGE_ALLOWED_TWO_STEP_VERIFICATION_METHODS", - "CHANGE_API_ACCESS", - "CHANGE_APPLICATION_SETTING", - "CHANGE_APP_ACCESS_SETTINGS_COLLECTION_ID", - "CHANGE_CALENDAR_SETTING", - "CHANGE_CHAT_SETTING", - "CHANGE_CHROME_OS_ANDROID_APPLICATION_SETTING", - "CHANGE_CHROME_OS_APPLICATION_SETTING", - "CHANGE_CHROME_OS_DEVICE_SETTING", - "CHANGE_CHROME_OS_PUBLIC_SESSION_SETTING", - "CHANGE_CHROME_OS_SETTING", - "CHANGE_CHROME_OS_USER_SETTING", - "CHANGE_CONTACTS_SETTING", - "CHANGE_DATA_LOCALIZATION_SETTING", - "CHANGE_DOCS_SETTING", - "CHANGE_EMAIL_SETTING", - "CHANGE_GMAIL_SETTING", - "CHANGE_MOBILE_APPLICATION_SETTINGS", - "CHANGE_MOBILE_SETTING", - "CHANGE_SESSION_LENGTH", - "CHANGE_SITES_SETTING", - "CHANGE_SITES_WEB_ADDRESS_MAPPING_UPDATES", - "CHANGE_SSO_SETTINGS", - "CHANGE_TWO_STEP_VERIFICATION_ENROLLMENT_PERIOD_DURATION", - "CHANGE_TWO_STEP_VERIFICATION_FREQUENCY", - "CHANGE_TWO_STEP_VERIFICATION_GRACE_PERIOD_DURATION", - "CHANGE_TWO_STEP_VERIFICATION_START_DATE", - "CHANGE_WHITELIST_SETTING", - "COMMUNICATION_PREFERENCES_SETTING_CHANGE", - "DELETE_APPLICATION_SETTING", - "DELETE_GMAIL_SETTING", - "DISALLOW_SERVICE_FOR_OAUTH2_ACCESS", - "ENABLE_API_ACCESS", - "ENABLE_FEEDBACK_SOLICITATION", - "ENABLE_NON_ADMIN_USER_PASSWORD_RECOVERY", - "ENABLE_SERVICE_OR_FEATURE_NOTIFICATIONS", - "ENFORCE_STRONG_AUTHENTICATION", - "FLASHLIGHT_EDU_NON_FEATURED_SERVICES_SELECTED", - "GPLUS_PREMIUM_FEATURES", - "MEET_INTEROP_MODIFY_GATEWAY", - "SESSION_CONTROL_SETTINGS_CHANGE", - "TOGGLE_ALLOW_ADMIN_PASSWORD_RESET", - "TOGGLE_CONTACT_SHARING", - "TOGGLE_ENABLE_OAUTH_CONSUMER_KEY", - "TOGGLE_NEW_APP_FEATURES", - "TOGGLE_OAUTH_ACCESS_TO_ALL_APIS", - "TOGGLE_OPEN_ID_ENABLED", - "TOGGLE_OUTBOUND_RELAY", - "TOGGLE_SSL", - "TOGGLE_SSO_ENABLED", - "TOGGLE_USE_CUSTOM_LOGO", - "TOGGLE_USE_NEXT_GEN_CONTROL_PANEL", - "UPDATE_CALENDAR_RESOURCE_FEATURE", - "UPDATE_ERROR_MSG_FOR_RESTRICTED_OAUTH2_APPS", - "UPDATE_MANAGED_CONFIGURATION", - "WEAK_PROGRAMMATIC_LOGIN_SETTINGS_CHANGED" - ].contains(ctx?.event?.action)' + if: '[ "ADMIN_EVENTS_TOGGLE_NEW_APP_FEATURES_PREFERENCE", "ALLOW_SERVICE_FOR_OAUTH2_ACCESS", "ALLOW_STRONG_AUTHENTICATION", "CHANGE_ALLOWED_TWO_STEP_VERIFICATION_METHODS", "CHANGE_API_ACCESS", "CHANGE_APPLICATION_SETTING", "CHANGE_APP_ACCESS_SETTINGS_COLLECTION_ID", "CHANGE_CALENDAR_SETTING", "CHANGE_CHAT_SETTING", "CHANGE_CHROME_OS_ANDROID_APPLICATION_SETTING", "CHANGE_CHROME_OS_APPLICATION_SETTING", "CHANGE_CHROME_OS_DEVICE_SETTING", "CHANGE_CHROME_OS_PUBLIC_SESSION_SETTING", "CHANGE_CHROME_OS_SETTING", "CHANGE_CHROME_OS_USER_SETTING", "CHANGE_CONTACTS_SETTING", "CHANGE_DATA_LOCALIZATION_SETTING", "CHANGE_DOCS_SETTING", "CHANGE_EMAIL_SETTING", "CHANGE_GMAIL_SETTING", "CHANGE_MOBILE_APPLICATION_SETTINGS", "CHANGE_MOBILE_SETTING", "CHANGE_SESSION_LENGTH", "CHANGE_SITES_SETTING", "CHANGE_SITES_WEB_ADDRESS_MAPPING_UPDATES", "CHANGE_SSO_SETTINGS", "CHANGE_TWO_STEP_VERIFICATION_ENROLLMENT_PERIOD_DURATION", "CHANGE_TWO_STEP_VERIFICATION_FREQUENCY", "CHANGE_TWO_STEP_VERIFICATION_GRACE_PERIOD_DURATION", "CHANGE_TWO_STEP_VERIFICATION_START_DATE", "CHANGE_WHITELIST_SETTING", "COMMUNICATION_PREFERENCES_SETTING_CHANGE", "DELETE_APPLICATION_SETTING", "DELETE_GMAIL_SETTING", "DISALLOW_SERVICE_FOR_OAUTH2_ACCESS", "ENABLE_API_ACCESS", "ENABLE_FEEDBACK_SOLICITATION", "ENABLE_NON_ADMIN_USER_PASSWORD_RECOVERY", "ENABLE_SERVICE_OR_FEATURE_NOTIFICATIONS", "ENFORCE_STRONG_AUTHENTICATION", "FLASHLIGHT_EDU_NON_FEATURED_SERVICES_SELECTED", "GPLUS_PREMIUM_FEATURES", "MEET_INTEROP_MODIFY_GATEWAY", "SESSION_CONTROL_SETTINGS_CHANGE", "TOGGLE_ALLOW_ADMIN_PASSWORD_RESET", "TOGGLE_CONTACT_SHARING", "TOGGLE_ENABLE_OAUTH_CONSUMER_KEY", "TOGGLE_NEW_APP_FEATURES", "TOGGLE_OAUTH_ACCESS_TO_ALL_APIS", "TOGGLE_OPEN_ID_ENABLED", "TOGGLE_OUTBOUND_RELAY", "TOGGLE_SSL", "TOGGLE_SSO_ENABLED", "TOGGLE_USE_CUSTOM_LOGO", "TOGGLE_USE_NEXT_GEN_CONTROL_PANEL", "UPDATE_CALENDAR_RESOURCE_FEATURE", "UPDATE_ERROR_MSG_FOR_RESTRICTED_OAUTH2_APPS", "UPDATE_MANAGED_CONFIGURATION", "WEAK_PROGRAMMATIC_LOGIN_SETTINGS_CHANGED" ].contains(ctx?.event?.action)' + - append: + tag: append_event_type_0814ff76 field: event.type value: change - if: '[ - "ADD_APPLICATION", - "ADD_APPLICATION_TO_WHITELIST", - "ADD_DOMAIN_ALIAS", - "ADD_GROUP_MEMBER", - "ADD_MOBILE_APPLICATION_TO_WHITELIST", - "ADD_MOBILE_CERTIFICATE", - "ADD_MOBILE_WIRELESS_NETWORK", - "ADD_NICKNAME", - "ADD_PRIVILEGE", - "ADD_RECOVERY_EMAIL", - "ADD_RECOVERY_PHONE", - "ADD_SECONDARY_DOMAIN", - "ADD_TO_TRUSTED_OAUTH2_APPS", - "ADD_TRUSTED_DOMAINS", - "ADMIN_EVENTS_TOGGLE_NEW_APP_FEATURES_PREFERENCE", - "ALERT_RECEIVERS_CHANGED", - "ALERT_STATUS_CHANGED", - "ALLOW_SERVICE_FOR_OAUTH2_ACCESS", - "ALLOW_STRONG_AUTHENTICATION", - "ARCHIVE_USER", - "ASSIGN_CUSTOM_LOGO", - "ASSIGN_ROLE", - "AUTHORIZE_API_CLIENT_ACCESS", - "BLOCK_ON_DEVICE_ACCESS", - "CANCEL_CALENDAR_EVENTS", - "CANCEL_USER_INVITE", - "CHANGE_ACCOUNT_AUTO_RENEWAL", - "CHANGE_ADMIN_RESTRICTIONS_PIN", - "CHANGE_ADVERTISEMENT_OPTION", - "CHANGE_ALERT_CRITERIA", - "CHANGE_ALLOWED_TWO_STEP_VERIFICATION_METHODS", - "CHANGE_API_ACCESS", - "CHANGE_APPLICATION_SETTING", - "CHANGE_APP_ACCESS", - "CHANGE_APP_ACCESS_SETTINGS_COLLECTION_ID", - "CHANGE_CAA_APP_ASSIGNMENTS", - "CHANGE_CAA_ERROR_MESSAGE", - "CHANGE_CALENDAR_SETTING", - "CHANGE_CHAT_SETTING", - "CHANGE_CHROME_OS_ANDROID_APPLICATION_SETTING", - "CHANGE_CHROME_OS_APPLICATION_SETTING", - "CHANGE_CHROME_OS_DEVICE_ANNOTATION", - "CHANGE_CHROME_OS_DEVICE_SETTING", - "CHANGE_CHROME_OS_DEVICE_STATE", - "CHANGE_CHROME_OS_PUBLIC_SESSION_SETTING", - "CHANGE_CHROME_OS_SETTING", - "CHANGE_CHROME_OS_USER_SETTING", - "CHANGE_CONFLICT_ACCOUNT_ACTION", - "CHANGE_CONTACTS_SETTING", - "CHANGE_CUSTOM_LOGO", - "CHANGE_DATA_LOCALIZATION_FOR_RUSSIA", - "CHANGE_DATA_LOCALIZATION_SETTING", - "CHANGE_DATA_PROTECTION_OFFICER_CONTACT_INFO", - "CHANGE_DEVICE_STATE", - "CHANGE_DOCS_SETTING", - "CHANGE_DOMAIN_DEFAULT_LOCALE", - "CHANGE_DOMAIN_DEFAULT_TIMEZONE", - "CHANGE_DOMAIN_NAME", - "CHANGE_DOMAIN_SUPPORT_MESSAGE", - "CHANGE_EDU_TYPE", - "CHANGE_EMAIL_SETTING", - "CHANGE_EU_REPRESENTATIVE_CONTACT_INFO", - "CHANGE_FIRST_NAME", - "CHANGE_GMAIL_SETTING", - "CHANGE_GROUP_DESCRIPTION", - "CHANGE_GROUP_NAME", - "CHANGE_GROUP_SETTING", - "CHANGE_LAST_NAME", - "CHANGE_LICENSE_AUTO_ASSIGN", - "CHANGE_LOGIN_ACTIVITY_TRACE", - "CHANGE_LOGIN_BACKGROUND_COLOR", - "CHANGE_LOGIN_BORDER_COLOR", - "CHANGE_MOBILE_APPLICATION_PERMISSION_GRANT", - "CHANGE_MOBILE_APPLICATION_PRIORITY_ORDER", - "CHANGE_MOBILE_APPLICATION_SETTINGS", - "CHANGE_MOBILE_SETTING", - "CHANGE_MOBILE_WIRELESS_NETWORK", - "CHANGE_MOBILE_WIRELESS_NETWORK_PASSWORD", - "CHANGE_ORGANIZATION_NAME", - "CHANGE_PASSWORD", - "CHANGE_PASSWORD_MAX_LENGTH", - "CHANGE_PASSWORD_MIN_LENGTH", - "CHANGE_PASSWORD_ON_NEXT_LOGIN", - "CHANGE_PRIMARY_DOMAIN", - "CHANGE_RECOVERY_EMAIL", - "CHANGE_RECOVERY_PHONE", - "CHANGE_RENEW_DOMAIN_REGISTRATION", - "CHANGE_RESELLER_ACCESS", - "CHANGE_RULE_CRITERIA", - "CHANGE_SESSION_LENGTH", - "CHANGE_SITES_SETTING", - "CHANGE_SITES_WEB_ADDRESS_MAPPING_UPDATES", - "CHANGE_SSO_SETTINGS", - "CHANGE_TWO_STEP_VERIFICATION_ENROLLMENT_PERIOD_DURATION", - "CHANGE_TWO_STEP_VERIFICATION_FREQUENCY", - "CHANGE_TWO_STEP_VERIFICATION_GRACE_PERIOD_DURATION", - "CHANGE_TWO_STEP_VERIFICATION_START_DATE", - "CHANGE_UNCONFIGURED_APPS_ACCESS", - "CHANGE_UNDERAGE_UNCONFIGURED_APPS_ACCESS", - "CHANGE_USER_ADDRESS", - "CHANGE_USER_CUSTOM_FIELD", - "CHANGE_USER_EXTERNAL_ID", - "CHANGE_USER_GENDER", - "CHANGE_USER_IM", - "CHANGE_USER_KEYWORD", - "CHANGE_USER_LANGUAGE", - "CHANGE_USER_LOCATION", - "CHANGE_USER_ORGANIZATION", - "CHANGE_USER_PHONE_NUMBER", - "CHANGE_USER_RELATION", - "CHANGE_WHITELIST_SETTING", - "CHROME_APPLICATION_LICENSE_RESERVATION_UPDATED", - "CHROME_LICENSES_ALLOWED", - "CHROME_LICENSES_ENABLED", - "CHROME_LICENSES_REDEEMED", - "COMMUNICATION_PREFERENCES_SETTING_CHANGE", - "COMPANY_OWNED_DEVICE_BLOCKED", - "COMPANY_OWNED_DEVICE_UNBLOCKED", - "COMPANY_OWNED_DEVICE_WIPED", - "CREATE_APPLICATION_SETTING", - "CREATE_GMAIL_SETTING", - "DISALLOW_SERVICE_FOR_OAUTH2_ACCESS", - "DOWNGRADE_USER_FROM_GPLUS", - "DROP_FROM_QUARANTINE", - "EDIT_ORG_UNIT_DESCRIPTION", - "EDIT_ORG_UNIT_NAME", - "ENABLE_API_ACCESS", - "ENABLE_FEEDBACK_SOLICITATION", - "ENABLE_NON_ADMIN_USER_PASSWORD_RECOVERY", - "ENABLE_SERVICE_OR_FEATURE_NOTIFICATIONS", - "ENABLE_USER_IP_WHITELIST", - "ENFORCE_STRONG_AUTHENTICATION", - "FLASHLIGHT_EDU_NON_FEATURED_SERVICES_SELECTED", - "GMAIL_RESET_USER", - "GPLUS_PREMIUM_FEATURES", - "GRANT_ADMIN_PRIVILEGE", - "GRANT_DELEGATED_ADMIN_PRIVILEGES", - "GROUP_MEMBER_BULK_UPLOAD", - "MAIL_ROUTING_DESTINATION_ADDED", - "MAIL_ROUTING_DESTINATION_REMOVED", - "MEET_INTEROP_MODIFY_GATEWAY", - "MOBILE_ACCOUNT_WIPE", - "MOBILE_DEVICE_APPROVE", - "MOBILE_DEVICE_BLOCK", - "MOBILE_DEVICE_CANCEL_WIPE_THEN_APPROVE", - "MOBILE_DEVICE_CANCEL_WIPE_THEN_BLOCK", - "MOBILE_DEVICE_WIPE", - "MOVE_DEVICE_TO_ORG_UNIT_DETAILED", - "MOVE_ORG_UNIT", - "MOVE_USER_TO_ORG_UNIT", - "ORG_ALL_USERS_LICENSE_ASSIGNMENT", - "ORG_LICENSE_REVOKE", - "ORG_USERS_LICENSE_ASSIGNMENT", - "PLAY_FOR_WORK_ENROLL", - "PLAY_FOR_WORK_UNENROLL", - "REJECT_FROM_QUARANTINE", - "RELEASE_CALENDAR_RESOURCES", - "RELEASE_FROM_QUARANTINE", - "REMOVE_API_CLIENT_ACCESS", - "REMOVE_APPLICATION", - "REMOVE_APPLICATION_FROM_WHITELIST", - "REMOVE_DOMAIN_ALIAS", - "REMOVE_FROM_TRUSTED_OAUTH2_APPS", - "REMOVE_GROUP_MEMBER", - "REMOVE_MOBILE_APPLICATION_FROM_WHITELIST", - "REMOVE_MOBILE_CERTIFICATE", - "REMOVE_MOBILE_WIRELESS_NETWORK", - "REMOVE_NICKNAME", - "REMOVE_PRIVILEGE", - "REMOVE_RECOVERY_EMAIL", - "REMOVE_RECOVERY_PHONE", - "REMOVE_SECONDARY_DOMAIN", - "REMOVE_TRUSTED_DOMAINS", - "RENAME_ALERT", - "RENAME_CALENDAR_RESOURCE", - "RENAME_ROLE", - "RENAME_RULE", - "RENAME_USER", - "REORDER_GROUP_BASED_POLICIES_EVENT", - "RESET_SIGNIN_COOKIES", - "REVOKE_3LO_DEVICE_TOKENS", - "REVOKE_3LO_TOKEN", - "REVOKE_ADMIN_PRIVILEGE", - "REVOKE_ASP", - "REVOKE_DEVICE_ENROLLMENT_TOKEN", - "REVOKE_ENROLLMENT_TOKEN", - "REVOKE_SECURITY_KEY", - "RULE_ACTIONS_CHANGED", - "RULE_STATUS_CHANGED", - "SECURITY_KEY_REGISTERED_FOR_USER", - "SEND_CHROME_OS_DEVICE_COMMAND", - "SESSION_CONTROL_SETTINGS_CHANGE", - "SUSPEND_USER", - "TOGGLE_ALLOW_ADMIN_PASSWORD_RESET", - "TOGGLE_AUTOMATIC_CONTACT_SHARING", - "TOGGLE_AUTO_ADD_NEW_SERVICE", - "TOGGLE_CAA_ENABLEMENT", - "TOGGLE_CONTACT_SHARING", - "TOGGLE_ENABLE_OAUTH_CONSUMER_KEY", - "TOGGLE_ENABLE_PRE_RELEASE_FEATURES", - "TOGGLE_NEW_APP_FEATURES", - "TOGGLE_OAUTH_ACCESS_TO_ALL_APIS", - "TOGGLE_OPEN_ID_ENABLED", - "TOGGLE_OUTBOUND_RELAY", - "TOGGLE_SERVICE_ENABLED", - "TOGGLE_SSL", - "TOGGLE_SSO_ENABLED", - "TOGGLE_USE_CUSTOM_LOGO", - "TOGGLE_USE_NEXT_GEN_CONTROL_PANEL", - "TRANSFER_DOCUMENT_OWNERSHIP", - "TRUST_DOMAIN_OWNED_OAUTH2_APPS", - "TURN_OFF_2_STEP_VERIFICATION", - "UNARCHIVE_USER", - "UNASSIGN_CUSTOM_LOGO", - "UNASSIGN_ROLE", - "UNBLOCK_ON_DEVICE_ACCESS", - "UNBLOCK_USER_SESSION", - "UNENROLL_USER_FROM_STRONG_AUTH", - "UNENROLL_USER_FROM_TITANIUM", - "UNSUSPEND_USER", - "UNTRUST_DOMAIN_OWNED_OAUTH2_APPS", - "UPDATE_BIRTHDATE", - "UPDATE_BUILDING", - "UPDATE_CALENDAR_RESOURCE", - "UPDATE_CALENDAR_RESOURCE_FEATURE", - "UPDATE_CHROME_OS_PRINTER", - "UPDATE_CHROME_OS_PRINT_SERVER", - "UPDATE_DEVICE", - "UPDATE_DOMAIN_PRIMARY_ADMIN_EMAIL", - "UPDATE_DOMAIN_SECONDARY_EMAIL", - "UPDATE_DYNAMIC_LICENSE", - "UPDATE_ERROR_MSG_FOR_RESTRICTED_OAUTH2_APPS", - "UPDATE_GROUP_MEMBER", - "UPDATE_GROUP_MEMBER_DELIVERY_SETTINGS", - "UPDATE_GROUP_MEMBER_DELIVERY_SETTINGS_CAN_EMAIL_OVERRIDE", - "UPDATE_MANAGED_CONFIGURATION", - "UPDATE_ROLE", - "UPDATE_RULE", - "UPGRADE_USER_TO_GPLUS", - "USER_ENROLLED_IN_TWO_STEP_VERIFICATION", - "USER_LICENSE_ASSIGNMENT", - "USER_LICENSE_REASSIGNMENT", - "USER_LICENSE_REVOKE", - "USER_PUT_IN_TWO_STEP_VERIFICATION_GRACE_PERIOD", - "WEAK_PROGRAMMATIC_LOGIN_SETTINGS_CHANGED", - "WHITELISTED_GROUPS_UPDATED" - ].contains(ctx?.event?.action)' + if: '[ "ADD_APPLICATION", "ADD_APPLICATION_TO_WHITELIST", "ADD_DOMAIN_ALIAS", "ADD_GROUP_MEMBER", "ADD_MOBILE_APPLICATION_TO_WHITELIST", "ADD_MOBILE_CERTIFICATE", "ADD_MOBILE_WIRELESS_NETWORK", "ADD_NICKNAME", "ADD_PRIVILEGE", "ADD_RECOVERY_EMAIL", "ADD_RECOVERY_PHONE", "ADD_SECONDARY_DOMAIN", "ADD_TO_TRUSTED_OAUTH2_APPS", "ADD_TRUSTED_DOMAINS", "ADMIN_EVENTS_TOGGLE_NEW_APP_FEATURES_PREFERENCE", "ALERT_RECEIVERS_CHANGED", "ALERT_STATUS_CHANGED", "ALLOW_SERVICE_FOR_OAUTH2_ACCESS", "ALLOW_STRONG_AUTHENTICATION", "ARCHIVE_USER", "ASSIGN_CUSTOM_LOGO", "ASSIGN_ROLE", "AUTHORIZE_API_CLIENT_ACCESS", "BLOCK_ON_DEVICE_ACCESS", "CANCEL_CALENDAR_EVENTS", "CANCEL_USER_INVITE", "CHANGE_ACCOUNT_AUTO_RENEWAL", "CHANGE_ADMIN_RESTRICTIONS_PIN", "CHANGE_ADVERTISEMENT_OPTION", "CHANGE_ALERT_CRITERIA", "CHANGE_ALLOWED_TWO_STEP_VERIFICATION_METHODS", "CHANGE_API_ACCESS", "CHANGE_APPLICATION_SETTING", "CHANGE_APP_ACCESS", "CHANGE_APP_ACCESS_SETTINGS_COLLECTION_ID", "CHANGE_CAA_APP_ASSIGNMENTS", "CHANGE_CAA_ERROR_MESSAGE", "CHANGE_CALENDAR_SETTING", "CHANGE_CHAT_SETTING", "CHANGE_CHROME_OS_ANDROID_APPLICATION_SETTING", "CHANGE_CHROME_OS_APPLICATION_SETTING", "CHANGE_CHROME_OS_DEVICE_ANNOTATION", "CHANGE_CHROME_OS_DEVICE_SETTING", "CHANGE_CHROME_OS_DEVICE_STATE", "CHANGE_CHROME_OS_PUBLIC_SESSION_SETTING", "CHANGE_CHROME_OS_SETTING", "CHANGE_CHROME_OS_USER_SETTING", "CHANGE_CONFLICT_ACCOUNT_ACTION", "CHANGE_CONTACTS_SETTING", "CHANGE_CUSTOM_LOGO", "CHANGE_DATA_LOCALIZATION_FOR_RUSSIA", "CHANGE_DATA_LOCALIZATION_SETTING", "CHANGE_DATA_PROTECTION_OFFICER_CONTACT_INFO", "CHANGE_DEVICE_STATE", "CHANGE_DOCS_SETTING", "CHANGE_DOMAIN_DEFAULT_LOCALE", "CHANGE_DOMAIN_DEFAULT_TIMEZONE", "CHANGE_DOMAIN_NAME", "CHANGE_DOMAIN_SUPPORT_MESSAGE", "CHANGE_EDU_TYPE", "CHANGE_EMAIL_SETTING", "CHANGE_EU_REPRESENTATIVE_CONTACT_INFO", "CHANGE_FIRST_NAME", "CHANGE_GMAIL_SETTING", "CHANGE_GROUP_DESCRIPTION", "CHANGE_GROUP_NAME", "CHANGE_GROUP_SETTING", "CHANGE_LAST_NAME", "CHANGE_LICENSE_AUTO_ASSIGN", "CHANGE_LOGIN_ACTIVITY_TRACE", "CHANGE_LOGIN_BACKGROUND_COLOR", "CHANGE_LOGIN_BORDER_COLOR", "CHANGE_MOBILE_APPLICATION_PERMISSION_GRANT", "CHANGE_MOBILE_APPLICATION_PRIORITY_ORDER", "CHANGE_MOBILE_APPLICATION_SETTINGS", "CHANGE_MOBILE_SETTING", "CHANGE_MOBILE_WIRELESS_NETWORK", "CHANGE_MOBILE_WIRELESS_NETWORK_PASSWORD", "CHANGE_ORGANIZATION_NAME", "CHANGE_PASSWORD", "CHANGE_PASSWORD_MAX_LENGTH", "CHANGE_PASSWORD_MIN_LENGTH", "CHANGE_PASSWORD_ON_NEXT_LOGIN", "CHANGE_PRIMARY_DOMAIN", "CHANGE_RECOVERY_EMAIL", "CHANGE_RECOVERY_PHONE", "CHANGE_RENEW_DOMAIN_REGISTRATION", "CHANGE_RESELLER_ACCESS", "CHANGE_RULE_CRITERIA", "CHANGE_SESSION_LENGTH", "CHANGE_SITES_SETTING", "CHANGE_SITES_WEB_ADDRESS_MAPPING_UPDATES", "CHANGE_SSO_SETTINGS", "CHANGE_TWO_STEP_VERIFICATION_ENROLLMENT_PERIOD_DURATION", "CHANGE_TWO_STEP_VERIFICATION_FREQUENCY", "CHANGE_TWO_STEP_VERIFICATION_GRACE_PERIOD_DURATION", "CHANGE_TWO_STEP_VERIFICATION_START_DATE", "CHANGE_UNCONFIGURED_APPS_ACCESS", "CHANGE_UNDERAGE_UNCONFIGURED_APPS_ACCESS", "CHANGE_USER_ADDRESS", "CHANGE_USER_CUSTOM_FIELD", "CHANGE_USER_EXTERNAL_ID", "CHANGE_USER_GENDER", "CHANGE_USER_IM", "CHANGE_USER_KEYWORD", "CHANGE_USER_LANGUAGE", "CHANGE_USER_LOCATION", "CHANGE_USER_ORGANIZATION", "CHANGE_USER_PHONE_NUMBER", "CHANGE_USER_RELATION", "CHANGE_WHITELIST_SETTING", "CHROME_APPLICATION_LICENSE_RESERVATION_UPDATED", "CHROME_LICENSES_ALLOWED", "CHROME_LICENSES_ENABLED", "CHROME_LICENSES_REDEEMED", "COMMUNICATION_PREFERENCES_SETTING_CHANGE", "COMPANY_OWNED_DEVICE_BLOCKED", "COMPANY_OWNED_DEVICE_UNBLOCKED", "COMPANY_OWNED_DEVICE_WIPED", "CREATE_APPLICATION_SETTING", "CREATE_GMAIL_SETTING", "DISALLOW_SERVICE_FOR_OAUTH2_ACCESS", "DOWNGRADE_USER_FROM_GPLUS", "DROP_FROM_QUARANTINE", "EDIT_ORG_UNIT_DESCRIPTION", "EDIT_ORG_UNIT_NAME", "ENABLE_API_ACCESS", "ENABLE_FEEDBACK_SOLICITATION", "ENABLE_NON_ADMIN_USER_PASSWORD_RECOVERY", "ENABLE_SERVICE_OR_FEATURE_NOTIFICATIONS", "ENABLE_USER_IP_WHITELIST", "ENFORCE_STRONG_AUTHENTICATION", "FLASHLIGHT_EDU_NON_FEATURED_SERVICES_SELECTED", "GMAIL_RESET_USER", "GPLUS_PREMIUM_FEATURES", "GRANT_ADMIN_PRIVILEGE", "GRANT_DELEGATED_ADMIN_PRIVILEGES", "GROUP_MEMBER_BULK_UPLOAD", "MAIL_ROUTING_DESTINATION_ADDED", "MAIL_ROUTING_DESTINATION_REMOVED", "MEET_INTEROP_MODIFY_GATEWAY", "MOBILE_ACCOUNT_WIPE", "MOBILE_DEVICE_APPROVE", "MOBILE_DEVICE_BLOCK", "MOBILE_DEVICE_CANCEL_WIPE_THEN_APPROVE", "MOBILE_DEVICE_CANCEL_WIPE_THEN_BLOCK", "MOBILE_DEVICE_WIPE", "MOVE_DEVICE_TO_ORG_UNIT_DETAILED", "MOVE_ORG_UNIT", "MOVE_USER_TO_ORG_UNIT", "ORG_ALL_USERS_LICENSE_ASSIGNMENT", "ORG_LICENSE_REVOKE", "ORG_USERS_LICENSE_ASSIGNMENT", "PLAY_FOR_WORK_ENROLL", "PLAY_FOR_WORK_UNENROLL", "REJECT_FROM_QUARANTINE", "RELEASE_CALENDAR_RESOURCES", "RELEASE_FROM_QUARANTINE", "REMOVE_API_CLIENT_ACCESS", "REMOVE_APPLICATION", "REMOVE_APPLICATION_FROM_WHITELIST", "REMOVE_DOMAIN_ALIAS", "REMOVE_FROM_TRUSTED_OAUTH2_APPS", "REMOVE_GROUP_MEMBER", "REMOVE_MOBILE_APPLICATION_FROM_WHITELIST", "REMOVE_MOBILE_CERTIFICATE", "REMOVE_MOBILE_WIRELESS_NETWORK", "REMOVE_NICKNAME", "REMOVE_PRIVILEGE", "REMOVE_RECOVERY_EMAIL", "REMOVE_RECOVERY_PHONE", "REMOVE_SECONDARY_DOMAIN", "REMOVE_TRUSTED_DOMAINS", "RENAME_ALERT", "RENAME_CALENDAR_RESOURCE", "RENAME_ROLE", "RENAME_RULE", "RENAME_USER", "REORDER_GROUP_BASED_POLICIES_EVENT", "RESET_SIGNIN_COOKIES", "REVOKE_3LO_DEVICE_TOKENS", "REVOKE_3LO_TOKEN", "REVOKE_ADMIN_PRIVILEGE", "REVOKE_ASP", "REVOKE_DEVICE_ENROLLMENT_TOKEN", "REVOKE_ENROLLMENT_TOKEN", "REVOKE_SECURITY_KEY", "RULE_ACTIONS_CHANGED", "RULE_STATUS_CHANGED", "SECURITY_KEY_REGISTERED_FOR_USER", "SEND_CHROME_OS_DEVICE_COMMAND", "SESSION_CONTROL_SETTINGS_CHANGE", "SUSPEND_USER", "TOGGLE_ALLOW_ADMIN_PASSWORD_RESET", "TOGGLE_AUTOMATIC_CONTACT_SHARING", "TOGGLE_AUTO_ADD_NEW_SERVICE", "TOGGLE_CAA_ENABLEMENT", "TOGGLE_CONTACT_SHARING", "TOGGLE_ENABLE_OAUTH_CONSUMER_KEY", "TOGGLE_ENABLE_PRE_RELEASE_FEATURES", "TOGGLE_NEW_APP_FEATURES", "TOGGLE_OAUTH_ACCESS_TO_ALL_APIS", "TOGGLE_OPEN_ID_ENABLED", "TOGGLE_OUTBOUND_RELAY", "TOGGLE_SERVICE_ENABLED", "TOGGLE_SSL", "TOGGLE_SSO_ENABLED", "TOGGLE_USE_CUSTOM_LOGO", "TOGGLE_USE_NEXT_GEN_CONTROL_PANEL", "TRANSFER_DOCUMENT_OWNERSHIP", "TRUST_DOMAIN_OWNED_OAUTH2_APPS", "TURN_OFF_2_STEP_VERIFICATION", "UNARCHIVE_USER", "UNASSIGN_CUSTOM_LOGO", "UNASSIGN_ROLE", "UNBLOCK_ON_DEVICE_ACCESS", "UNBLOCK_USER_SESSION", "UNENROLL_USER_FROM_STRONG_AUTH", "UNENROLL_USER_FROM_TITANIUM", "UNSUSPEND_USER", "UNTRUST_DOMAIN_OWNED_OAUTH2_APPS", "UPDATE_BIRTHDATE", "UPDATE_BUILDING", "UPDATE_CALENDAR_RESOURCE", "UPDATE_CALENDAR_RESOURCE_FEATURE", "UPDATE_CHROME_OS_PRINTER", "UPDATE_CHROME_OS_PRINT_SERVER", "UPDATE_DEVICE", "UPDATE_DOMAIN_PRIMARY_ADMIN_EMAIL", "UPDATE_DOMAIN_SECONDARY_EMAIL", "UPDATE_DYNAMIC_LICENSE", "UPDATE_ERROR_MSG_FOR_RESTRICTED_OAUTH2_APPS", "UPDATE_GROUP_MEMBER", "UPDATE_GROUP_MEMBER_DELIVERY_SETTINGS", "UPDATE_GROUP_MEMBER_DELIVERY_SETTINGS_CAN_EMAIL_OVERRIDE", "UPDATE_MANAGED_CONFIGURATION", "UPDATE_ROLE", "UPDATE_RULE", "UPGRADE_USER_TO_GPLUS", "USER_ENROLLED_IN_TWO_STEP_VERIFICATION", "USER_LICENSE_ASSIGNMENT", "USER_LICENSE_REASSIGNMENT", "USER_LICENSE_REVOKE", "USER_PUT_IN_TWO_STEP_VERIFICATION_GRACE_PERIOD", "WEAK_PROGRAMMATIC_LOGIN_SETTINGS_CHANGED", "WHITELISTED_GROUPS_UPDATED" ].contains(ctx?.event?.action)' + - append: + tag: append_event_type_8e16e7ab field: event.type value: user - if: '[ - "ACTION_CANCELLED", - "ACTION_REQUESTED", - "ADD_NICKNAME", - "ADD_RECOVERY_EMAIL", - "ADD_RECOVERY_PHONE", - "ARCHIVE_USER", - "BULK_UPLOAD_NOTIFICATION_SENT", - "CANCEL_USER_INVITE", - "CHANGE_FIRST_NAME", - "CHANGE_LAST_NAME", - "CHANGE_PASSWORD", - "CHANGE_PASSWORD_ON_NEXT_LOGIN", - "CHANGE_RECOVERY_EMAIL", - "CHANGE_RECOVERY_PHONE", - "CHANGE_USER_ADDRESS", - "CHANGE_USER_CUSTOM_FIELD", - "CHANGE_USER_EXTERNAL_ID", - "CHANGE_USER_GENDER", - "CHANGE_USER_IM", - "CHANGE_USER_KEYWORD", - "CHANGE_USER_LANGUAGE", - "CHANGE_USER_LOCATION", - "CHANGE_USER_ORGANIZATION", - "CHANGE_USER_PHONE_NUMBER", - "CHANGE_USER_RELATION", - "CREATE_DATA_TRANSFER_REQUEST", - "CREATE_EMAIL_MONITOR", - "CREATE_USER", - "DELETE_2SV_SCRATCH_CODES", - "DELETE_ACCOUNT_INFO_DUMP", - "DELETE_EMAIL_MONITOR", - "DELETE_MAILBOX_DUMP", - "DELETE_USER", - "DOWNGRADE_USER_FROM_GPLUS", - "ENABLE_USER_IP_WHITELIST", - "GENERATE_2SV_SCRATCH_CODES", - "GMAIL_RESET_USER", - "GRANT_ADMIN_PRIVILEGE", - "GRANT_DELEGATED_ADMIN_PRIVILEGES", - "MAIL_ROUTING_DESTINATION_ADDED", - "MAIL_ROUTING_DESTINATION_REMOVED", - "MOBILE_ACCOUNT_WIPE", - "MOBILE_DEVICE_APPROVE", - "MOBILE_DEVICE_BLOCK", - "MOBILE_DEVICE_CANCEL_WIPE_THEN_APPROVE", - "MOBILE_DEVICE_CANCEL_WIPE_THEN_BLOCK", - "MOBILE_DEVICE_DELETE", - "MOBILE_DEVICE_WIPE", - "MOVE_USER_TO_ORG_UNIT", - "REMOVE_NICKNAME", - "REMOVE_RECOVERY_EMAIL", - "REMOVE_RECOVERY_PHONE", - "RENAME_USER", - "REQUEST_ACCOUNT_INFO", - "REQUEST_MAILBOX_DUMP", - "RESEND_USER_INVITE", - "RESET_SIGNIN_COOKIES", - "REVOKE_3LO_DEVICE_TOKENS", - "REVOKE_3LO_TOKEN", - "REVOKE_ADMIN_PRIVILEGE", - "REVOKE_ASP", - "REVOKE_SECURITY_KEY", - "SECURITY_KEY_REGISTERED_FOR_USER", - "SUSPEND_USER", - "TOGGLE_AUTOMATIC_CONTACT_SHARING", - "TURN_OFF_2_STEP_VERIFICATION", - "UNARCHIVE_USER", - "UNBLOCK_USER_SESSION", - "UNDELETE_USER", - "UNENROLL_USER_FROM_STRONG_AUTH", - "UNENROLL_USER_FROM_TITANIUM", - "UNSUSPEND_USER", - "UPDATE_BIRTHDATE", - "UPGRADE_USER_TO_GPLUS", - "USERS_BULK_UPLOAD_NOTIFICATION_SENT", - "USER_ENROLLED_IN_TWO_STEP_VERIFICATION", - "USER_INVITE", - "USER_PUT_IN_TWO_STEP_VERIFICATION_GRACE_PERIOD", - "VIEW_TEMP_PASSWORD" - ].contains(ctx?.event?.action)' + if: '[ "ACTION_CANCELLED", "ACTION_REQUESTED", "ADD_NICKNAME", "ADD_RECOVERY_EMAIL", "ADD_RECOVERY_PHONE", "ARCHIVE_USER", "BULK_UPLOAD_NOTIFICATION_SENT", "CANCEL_USER_INVITE", "CHANGE_FIRST_NAME", "CHANGE_LAST_NAME", "CHANGE_PASSWORD", "CHANGE_PASSWORD_ON_NEXT_LOGIN", "CHANGE_RECOVERY_EMAIL", "CHANGE_RECOVERY_PHONE", "CHANGE_USER_ADDRESS", "CHANGE_USER_CUSTOM_FIELD", "CHANGE_USER_EXTERNAL_ID", "CHANGE_USER_GENDER", "CHANGE_USER_IM", "CHANGE_USER_KEYWORD", "CHANGE_USER_LANGUAGE", "CHANGE_USER_LOCATION", "CHANGE_USER_ORGANIZATION", "CHANGE_USER_PHONE_NUMBER", "CHANGE_USER_RELATION", "CREATE_DATA_TRANSFER_REQUEST", "CREATE_EMAIL_MONITOR", "CREATE_USER", "DELETE_2SV_SCRATCH_CODES", "DELETE_ACCOUNT_INFO_DUMP", "DELETE_EMAIL_MONITOR", "DELETE_MAILBOX_DUMP", "DELETE_USER", "DOWNGRADE_USER_FROM_GPLUS", "ENABLE_USER_IP_WHITELIST", "GENERATE_2SV_SCRATCH_CODES", "GMAIL_RESET_USER", "GRANT_ADMIN_PRIVILEGE", "GRANT_DELEGATED_ADMIN_PRIVILEGES", "MAIL_ROUTING_DESTINATION_ADDED", "MAIL_ROUTING_DESTINATION_REMOVED", "MOBILE_ACCOUNT_WIPE", "MOBILE_DEVICE_APPROVE", "MOBILE_DEVICE_BLOCK", "MOBILE_DEVICE_CANCEL_WIPE_THEN_APPROVE", "MOBILE_DEVICE_CANCEL_WIPE_THEN_BLOCK", "MOBILE_DEVICE_DELETE", "MOBILE_DEVICE_WIPE", "MOVE_USER_TO_ORG_UNIT", "REMOVE_NICKNAME", "REMOVE_RECOVERY_EMAIL", "REMOVE_RECOVERY_PHONE", "RENAME_USER", "REQUEST_ACCOUNT_INFO", "REQUEST_MAILBOX_DUMP", "RESEND_USER_INVITE", "RESET_SIGNIN_COOKIES", "REVOKE_3LO_DEVICE_TOKENS", "REVOKE_3LO_TOKEN", "REVOKE_ADMIN_PRIVILEGE", "REVOKE_ASP", "REVOKE_SECURITY_KEY", "SECURITY_KEY_REGISTERED_FOR_USER", "SUSPEND_USER", "TOGGLE_AUTOMATIC_CONTACT_SHARING", "TURN_OFF_2_STEP_VERIFICATION", "UNARCHIVE_USER", "UNBLOCK_USER_SESSION", "UNDELETE_USER", "UNENROLL_USER_FROM_STRONG_AUTH", "UNENROLL_USER_FROM_TITANIUM", "UNSUSPEND_USER", "UPDATE_BIRTHDATE", "UPGRADE_USER_TO_GPLUS", "USERS_BULK_UPLOAD_NOTIFICATION_SENT", "USER_ENROLLED_IN_TWO_STEP_VERIFICATION", "USER_INVITE", "USER_PUT_IN_TWO_STEP_VERIFICATION_GRACE_PERIOD", "VIEW_TEMP_PASSWORD" ].contains(ctx?.event?.action)' + - append: + tag: append_event_type_494fff08 field: event.type value: creation - if: '[ - "ADD_WEB_ADDRESS", - "CHROME_APPLICATION_LICENSE_RESERVATION_CREATED", - "COMPANY_DEVICES_BULK_CREATION", - "CREATE_ALERT", - "CREATE_APPLICATION_SETTING", - "CREATE_BUILDING", - "CREATE_CALENDAR_RESOURCE", - "CREATE_CALENDAR_RESOURCE_FEATURE", - "CREATE_DATA_TRANSFER_REQUEST", - "CREATE_DEVICE_ENROLLMENT_TOKEN", - "CREATE_EMAIL_MONITOR", - "CREATE_ENROLLMENT_TOKEN", - "CREATE_GMAIL_SETTING", - "CREATE_GROUP", - "CREATE_MANAGED_CONFIGURATION", - "CREATE_ORG_UNIT", - "CREATE_PLAY_FOR_WORK_TOKEN", - "CREATE_ROLE", - "CREATE_RULE", - "CREATE_USER", - "EMAIL_UNDELETE", - "GENERATE_2SV_SCRATCH_CODES", - "GENERATE_PIN", - "GENERATE_TRANSFER_TOKEN", - "INSERT_CHROME_OS_PRINTER", - "INSERT_CHROME_OS_PRINT_SERVER", - "MEET_INTEROP_CREATE_GATEWAY", - "REGENERATE_OAUTH_CONSUMER_SECRET", - "UNDELETE_USER" - ].contains(ctx?.event?.action)' + if: '[ "ADD_WEB_ADDRESS", "CHROME_APPLICATION_LICENSE_RESERVATION_CREATED", "COMPANY_DEVICES_BULK_CREATION", "CREATE_ALERT", "CREATE_APPLICATION_SETTING", "CREATE_BUILDING", "CREATE_CALENDAR_RESOURCE", "CREATE_CALENDAR_RESOURCE_FEATURE", "CREATE_DATA_TRANSFER_REQUEST", "CREATE_DEVICE_ENROLLMENT_TOKEN", "CREATE_EMAIL_MONITOR", "CREATE_ENROLLMENT_TOKEN", "CREATE_GMAIL_SETTING", "CREATE_GROUP", "CREATE_MANAGED_CONFIGURATION", "CREATE_ORG_UNIT", "CREATE_PLAY_FOR_WORK_TOKEN", "CREATE_ROLE", "CREATE_RULE", "CREATE_USER", "EMAIL_UNDELETE", "GENERATE_2SV_SCRATCH_CODES", "GENERATE_PIN", "GENERATE_TRANSFER_TOKEN", "INSERT_CHROME_OS_PRINTER", "INSERT_CHROME_OS_PRINT_SERVER", "MEET_INTEROP_CREATE_GATEWAY", "REGENERATE_OAUTH_CONSUMER_SECRET", "UNDELETE_USER" ].contains(ctx?.event?.action)' + - append: + tag: append_event_type_d6dc3868 field: event.type value: deletion - if: '[ - "CHROME_APPLICATION_LICENSE_RESERVATION_DELETED", - "COMPANY_DEVICE_DELETION", - "DELETE_2SV_SCRATCH_CODES", - "DELETE_ACCOUNT_INFO_DUMP", - "DELETE_ALERT", - "DELETE_APPLICATION_SETTING", - "DELETE_BUILDING", - "DELETE_CALENDAR_RESOURCE", - "DELETE_CALENDAR_RESOURCE_FEATURE", - "DELETE_CHROME_OS_PRINTER", - "DELETE_CHROME_OS_PRINT_SERVER", - "DELETE_EMAIL_MONITOR", - "DELETE_GMAIL_SETTING", - "DELETE_GROUP", - "DELETE_MAILBOX_DUMP", - "DELETE_MANAGED_CONFIGURATION", - "DELETE_PLAY_FOR_WORK_TOKEN", - "DELETE_ROLE", - "DELETE_RULE", - "DELETE_USER", - "DELETE_WEB_ADDRESS", - "MEET_INTEROP_DELETE_GATEWAY", - "MOBILE_DEVICE_DELETE", - "REMOVE_CHROME_OS_APPLICATION_SETTINGS", - "REMOVE_ORG_UNIT" - ].contains(ctx?.event?.action)' + if: '[ "CHROME_APPLICATION_LICENSE_RESERVATION_DELETED", "COMPANY_DEVICE_DELETION", "DELETE_2SV_SCRATCH_CODES", "DELETE_ACCOUNT_INFO_DUMP", "DELETE_ALERT", "DELETE_APPLICATION_SETTING", "DELETE_BUILDING", "DELETE_CALENDAR_RESOURCE", "DELETE_CALENDAR_RESOURCE_FEATURE", "DELETE_CHROME_OS_PRINTER", "DELETE_CHROME_OS_PRINT_SERVER", "DELETE_EMAIL_MONITOR", "DELETE_GMAIL_SETTING", "DELETE_GROUP", "DELETE_MAILBOX_DUMP", "DELETE_MANAGED_CONFIGURATION", "DELETE_PLAY_FOR_WORK_TOKEN", "DELETE_ROLE", "DELETE_RULE", "DELETE_USER", "DELETE_WEB_ADDRESS", "MEET_INTEROP_DELETE_GATEWAY", "MOBILE_DEVICE_DELETE", "REMOVE_CHROME_OS_APPLICATION_SETTINGS", "REMOVE_ORG_UNIT" ].contains(ctx?.event?.action)' + - append: + tag: append_event_type_83ac8b05 field: event.type value: group - if: '[ - "ADD_GROUP_MEMBER", - "CHANGE_GROUP_DESCRIPTION", - "CHANGE_GROUP_NAME", - "CHANGE_GROUP_SETTING", - "GROUP_LIST_DOWNLOAD", - "GROUP_MEMBERS_DOWNLOAD", - "GROUP_MEMBER_BULK_UPLOAD", - "REMOVE_GROUP_MEMBER", - "REORDER_GROUP_BASED_POLICIES_EVENT", - "UPDATE_GROUP_MEMBER", - "UPDATE_GROUP_MEMBER_DELIVERY_SETTINGS", - "UPDATE_GROUP_MEMBER_DELIVERY_SETTINGS_CAN_EMAIL_OVERRIDE", - "WHITELISTED_GROUPS_UPDATED" - ].contains(ctx?.event?.action)' + if: '[ "ADD_GROUP_MEMBER", "CHANGE_GROUP_DESCRIPTION", "CHANGE_GROUP_NAME", "CHANGE_GROUP_SETTING", "GROUP_LIST_DOWNLOAD", "GROUP_MEMBERS_DOWNLOAD", "GROUP_MEMBER_BULK_UPLOAD", "REMOVE_GROUP_MEMBER", "REORDER_GROUP_BASED_POLICIES_EVENT", "UPDATE_GROUP_MEMBER", "UPDATE_GROUP_MEMBER_DELIVERY_SETTINGS", "UPDATE_GROUP_MEMBER_DELIVERY_SETTINGS_CAN_EMAIL_OVERRIDE", "WHITELISTED_GROUPS_UPDATED" ].contains(ctx?.event?.action)' + - append: + tag: append_event_type_42b538d8 field: event.type value: info - if: '[ - "ACTION_CANCELLED", - "ACTION_REQUESTED", - "BULK_UPLOAD", - "BULK_UPLOAD_NOTIFICATION_SENT", - "DOWNLOAD_PENDING_INVITES_LIST", - "DOWNLOAD_USERLIST_CSV", - "DRIVE_DATA_RESTORE", - "EMAIL_LOG_SEARCH", - "ENROLL_FOR_GOOGLE_DEVICE_MANAGEMENT", - "GROUP_LIST_DOWNLOAD", - "GROUP_MEMBERS_DOWNLOAD", - "ISSUE_DEVICE_COMMAND", - "MX_RECORD_VERIFICATION_CLAIM", - "REQUEST_ACCOUNT_INFO", - "REQUEST_MAILBOX_DUMP", - "RESEND_USER_INVITE", - "SKIP_DOMAIN_ALIAS_MX", - "SKIP_SECONDARY_DOMAIN_MX", - "UPLOAD_OAUTH_CERTIFICATE", - "USERS_BULK_UPLOAD", - "USERS_BULK_UPLOAD_NOTIFICATION_SENT", - "USER_INVITE", - "USE_GOOGLE_MOBILE_MANAGEMENT", - "USE_GOOGLE_MOBILE_MANAGEMENT_FOR_IOS", - "USE_GOOGLE_MOBILE_MANAGEMENT_FOR_NON_IOS", - "VERIFY_DOMAIN_ALIAS", - "VERIFY_DOMAIN_ALIAS_MX", - "VERIFY_SECONDARY_DOMAIN", - "VERIFY_SECONDARY_DOMAIN_MX", - "VIEW_DNS_LOGIN_DETAILS", - "VIEW_SITE_DETAILS", - "VIEW_TEMP_PASSWORD" - ].contains(ctx?.event?.action)' + if: '[ "ACTION_CANCELLED", "ACTION_REQUESTED", "BULK_UPLOAD", "BULK_UPLOAD_NOTIFICATION_SENT", "DOWNLOAD_PENDING_INVITES_LIST", "DOWNLOAD_USERLIST_CSV", "DRIVE_DATA_RESTORE", "EMAIL_LOG_SEARCH", "ENROLL_FOR_GOOGLE_DEVICE_MANAGEMENT", "GROUP_LIST_DOWNLOAD", "GROUP_MEMBERS_DOWNLOAD", "ISSUE_DEVICE_COMMAND", "MX_RECORD_VERIFICATION_CLAIM", "REQUEST_ACCOUNT_INFO", "REQUEST_MAILBOX_DUMP", "RESEND_USER_INVITE", "SKIP_DOMAIN_ALIAS_MX", "SKIP_SECONDARY_DOMAIN_MX", "UPLOAD_OAUTH_CERTIFICATE", "USERS_BULK_UPLOAD", "USERS_BULK_UPLOAD_NOTIFICATION_SENT", "USER_INVITE", "USE_GOOGLE_MOBILE_MANAGEMENT", "USE_GOOGLE_MOBILE_MANAGEMENT_FOR_IOS", "USE_GOOGLE_MOBILE_MANAGEMENT_FOR_NON_IOS", "VERIFY_DOMAIN_ALIAS", "VERIFY_DOMAIN_ALIAS_MX", "VERIFY_SECONDARY_DOMAIN", "VERIFY_SECONDARY_DOMAIN_MX", "VIEW_DNS_LOGIN_DETAILS", "VIEW_SITE_DETAILS", "VIEW_TEMP_PASSWORD" ].contains(ctx?.event?.action)' + - script: + tag: script_105a5e8e lang: painless if: 'ctx?.json?.events?.parameters != null && ctx?.json?.events?.parameters instanceof List' source: > @@ -671,6 +219,7 @@ processors: } on_failure: - append: + tag: append_error_message_623fbe9c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -694,299 +243,374 @@ processors: target_field: google_workspace.admin.setting.metadata.user_defined.name ignore_missing: true - remove: + tag: remove_json_events_parameters_2eab3501 field: json.events.parameters ignore_missing: true - rename: + tag: rename_google_workspace_admin_APPLICATION_EDITION_to_google_workspace_admin_application_edition_701f4893 field: google_workspace.admin.APPLICATION_EDITION target_field: google_workspace.admin.application.edition ignore_missing: true - rename: + tag: rename_google_workspace_admin_APPLICATION_NAME_to_google_workspace_admin_application_name_10fd90bb field: google_workspace.admin.APPLICATION_NAME target_field: google_workspace.admin.application.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_APPLICATION_ENABLED_to_google_workspace_admin_application_enabled_eebb2d37 field: google_workspace.admin.APPLICATION_ENABLED target_field: google_workspace.admin.application.enabled ignore_missing: true - rename: + tag: rename_google_workspace_admin_APP_LICENSES_ORDER_NUMBER_to_google_workspace_admin_application_licences_order_number_3320e6b0 field: google_workspace.admin.APP_LICENSES_ORDER_NUMBER target_field: google_workspace.admin.application.licences_order_number ignore_missing: true - rename: + tag: rename_google_workspace_admin_CHROME_NUM_LICENSES_PURCHASED_to_google_workspace_admin_application_licences_purchased_633ca79e field: google_workspace.admin.CHROME_NUM_LICENSES_PURCHASED target_field: google_workspace.admin.application.licences_purchased ignore_missing: true - rename: + tag: rename_google_workspace_admin_REAUTH_APPLICATION_to_google_workspace_admin_application_name_d803fdb1 field: google_workspace.admin.REAUTH_APPLICATION target_field: google_workspace.admin.application.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_GROUP_EMAIL_to_google_workspace_admin_group_email_df5e2c63 field: google_workspace.admin.GROUP_EMAIL target_field: google_workspace.admin.group.email ignore_missing: true - rename: + tag: rename_google_workspace_admin_GROUP_NAME_to_group_name_c7bb67ed field: google_workspace.admin.GROUP_NAME target_field: group.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_NEW_VALUE_to_google_workspace_admin_new_value_69724b10 field: google_workspace.admin.NEW_VALUE target_field: google_workspace.admin.new_value ignore_missing: true - rename: + tag: rename_google_workspace_admin_OLD_VALUE_to_google_workspace_admin_old_value_d25e87c4 field: google_workspace.admin.OLD_VALUE target_field: google_workspace.admin.old_value ignore_missing: true - rename: + tag: rename_google_workspace_admin_ORG_UNIT_NAME_to_google_workspace_admin_org_unit_name_4e958d85 field: google_workspace.admin.ORG_UNIT_NAME target_field: google_workspace.admin.org_unit.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_SETTING_NAME_to_google_workspace_admin_setting_name_dc2a1cef field: google_workspace.admin.SETTING_NAME target_field: google_workspace.admin.setting.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_SETTING_DESCRIPTION_to_google_workspace_admin_setting_description_5f8febb7 field: google_workspace.admin.SETTING_DESCRIPTION target_field: google_workspace.admin.setting.description ignore_missing: true - rename: + tag: rename_google_workspace_admin_GROUP_PRIORITIES_to_google_workspace_admin_group_priorities_ab13f4cf field: google_workspace.admin.GROUP_PRIORITIES target_field: google_workspace.admin.group.priorities ignore_missing: true - rename: + tag: rename_google_workspace_admin_DOMAIN_NAME_to_google_workspace_admin_domain_name_c20fd251 field: google_workspace.admin.DOMAIN_NAME target_field: google_workspace.admin.domain.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_DOMAIN_ALIAS_to_google_workspace_admin_domain_alias_789028ab field: google_workspace.admin.DOMAIN_ALIAS target_field: google_workspace.admin.domain.alias ignore_missing: true - rename: + tag: rename_google_workspace_admin_SECONDARY_DOMAIN_NAME_to_google_workspace_admin_domain_secondary_name_6c1730cb field: google_workspace.admin.SECONDARY_DOMAIN_NAME target_field: google_workspace.admin.domain.secondary_name ignore_missing: true - rename: + tag: rename_google_workspace_admin_MANAGED_CONFIGURATION_NAME_to_google_workspace_admin_managed_configuration_2714f864 field: google_workspace.admin.MANAGED_CONFIGURATION_NAME target_field: google_workspace.admin.managed_configuration ignore_missing: true - rename: + tag: rename_google_workspace_admin_MOBILE_APP_PACKAGE_ID_to_google_workspace_admin_application_package_id_a3501123 field: google_workspace.admin.MOBILE_APP_PACKAGE_ID target_field: google_workspace.admin.application.package_id ignore_missing: true - rename: + tag: rename_google_workspace_admin_FLASHLIGHT_EDU_NON_FEATURED_SERVICES_SELECTION_to_google_workspace_admin_non_featured_services_selection_3d4d6278 field: google_workspace.admin.FLASHLIGHT_EDU_NON_FEATURED_SERVICES_SELECTION target_field: google_workspace.admin.non_featured_services_selection ignore_missing: true - rename: + tag: rename_google_workspace_admin_FIELD_NAME_to_google_workspace_admin_field_06dd8cf4 field: google_workspace.admin.FIELD_NAME target_field: google_workspace.admin.field ignore_missing: true - rename: + tag: rename_google_workspace_admin_RESOURCE_IDENTIFIER_to_google_workspace_admin_resource_id_1bd4176b field: google_workspace.admin.RESOURCE_IDENTIFIER target_field: google_workspace.admin.resource.id ignore_missing: true - rename: + tag: rename_google_workspace_admin_USER_EMAIL_to_google_workspace_admin_user_email_c9f46445 field: google_workspace.admin.USER_EMAIL target_field: google_workspace.admin.user.email ignore_missing: true - rename: + tag: rename_google_workspace_admin_GATEWAY_NAME_to_google_workspace_admin_gateway_name_4aa6ecbb field: google_workspace.admin.GATEWAY_NAME target_field: google_workspace.admin.gateway.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_APP_ID_to_google_workspace_admin_application_id_1c47e88c field: google_workspace.admin.APP_ID target_field: google_workspace.admin.application.id ignore_missing: true - rename: + tag: rename_google_workspace_admin_ASP_ID_to_google_workspace_admin_application_asp_id_f24e1fe6 field: google_workspace.admin.ASP_ID target_field: google_workspace.admin.application.asp_id ignore_missing: true - rename: + tag: rename_google_workspace_admin_CHROME_OS_SESSION_TYPE_to_google_workspace_admin_chrome_os_session_type_e16713a9 field: google_workspace.admin.CHROME_OS_SESSION_TYPE target_field: google_workspace.admin.chrome_os.session_type ignore_missing: true - rename: + tag: rename_google_workspace_admin_DEVICE_NEW_STATE_to_google_workspace_admin_new_value_5ef2c7e5 field: google_workspace.admin.DEVICE_NEW_STATE target_field: google_workspace.admin.new_value ignore_missing: true - rename: + tag: rename_google_workspace_admin_DEVICE_PREVIOUS_STATE_to_google_workspace_admin_old_value_3516a931 field: google_workspace.admin.DEVICE_PREVIOUS_STATE target_field: google_workspace.admin.old_value ignore_missing: true - rename: + tag: rename_google_workspace_admin_DEVICE_SERIAL_NUMBER_to_google_workspace_admin_device_serial_number_428bbf6f field: google_workspace.admin.DEVICE_SERIAL_NUMBER target_field: google_workspace.admin.device.serial_number ignore_missing: true - rename: + tag: rename_google_workspace_admin_DEVICE_ID_to_google_workspace_admin_device_id_11b73c91 field: google_workspace.admin.DEVICE_ID target_field: google_workspace.admin.device.id ignore_missing: true - rename: + tag: rename_google_workspace_admin_DEVICE_TYPE_to_google_workspace_admin_device_type_143b7b55 field: google_workspace.admin.DEVICE_TYPE target_field: google_workspace.admin.device.type ignore_missing: true - rename: + tag: rename_google_workspace_admin_PRINT_SERVER_NAME_to_google_workspace_admin_print_server_name_977bf33d field: google_workspace.admin.PRINT_SERVER_NAME target_field: google_workspace.admin.print_server.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_PRINTER_NAME_to_google_workspace_admin_printer_name_d839be4b field: google_workspace.admin.PRINTER_NAME target_field: google_workspace.admin.printer.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_DEVICE_COMMAND_DETAILS_to_google_workspace_admin_device_command_details_31edebf7 field: google_workspace.admin.DEVICE_COMMAND_DETAILS target_field: google_workspace.admin.device.command_details ignore_missing: true - rename: + tag: rename_google_workspace_admin_DEVICE_NEW_ORG_UNIT_to_google_workspace_admin_new_value_a712f0b7 field: google_workspace.admin.DEVICE_NEW_ORG_UNIT target_field: google_workspace.admin.new_value ignore_missing: true - rename: + tag: rename_google_workspace_admin_DEVICE_PREVIOUS_ORG_UNIT_to_google_workspace_admin_old_value_729609f1 field: google_workspace.admin.DEVICE_PREVIOUS_ORG_UNIT target_field: google_workspace.admin.old_value ignore_missing: true - rename: + tag: rename_google_workspace_admin_ROLE_NAME_to_google_workspace_admin_role_name_56641655 field: google_workspace.admin.ROLE_NAME target_field: google_workspace.admin.role.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_ROLE_ID_to_google_workspace_admin_role_id_f3ea0921 field: google_workspace.admin.ROLE_ID target_field: google_workspace.admin.role.id ignore_missing: true - rename: + tag: rename_google_workspace_admin_PRIVILEGE_NAME_to_google_workspace_admin_privilege_name_793d8379 field: google_workspace.admin.PRIVILEGE_NAME target_field: google_workspace.admin.privilege.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_SITE_LOCATION_to_url_path_3886800d field: google_workspace.admin.SITE_LOCATION target_field: url.path ignore_missing: true - rename: + tag: rename_google_workspace_admin_WEB_ADDRESS_to_url_full_d6af874d field: google_workspace.admin.WEB_ADDRESS target_field: url.full ignore_missing: true - uri_parts: + tag: uri_parts_url_full_fcf3baba field: url.full ignore_failure: true if: ctx?.url?.full != null - rename: + tag: rename_google_workspace_admin_SITE_NAME_to_google_workspace_admin_url_name_98f66d8d field: google_workspace.admin.SITE_NAME target_field: google_workspace.admin.url.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_SERVICE_NAME_to_google_workspace_admin_service_name_3a23250d field: google_workspace.admin.SERVICE_NAME target_field: google_workspace.admin.service.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_PRODUCT_NAME_to_google_workspace_admin_product_name_f34b1b81 field: google_workspace.admin.PRODUCT_NAME target_field: google_workspace.admin.product.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_SKU_NAME_to_google_workspace_admin_product_sku_ad320dd9 field: google_workspace.admin.SKU_NAME target_field: google_workspace.admin.product.sku ignore_missing: true - rename: + tag: rename_google_workspace_admin_GROUP_MEMBER_BULK_UPLOAD_FAILED_NUMBER_to_google_workspace_admin_bulk_upload_failed_46c188b2 field: google_workspace.admin.GROUP_MEMBER_BULK_UPLOAD_FAILED_NUMBER target_field: google_workspace.admin.bulk_upload.failed ignore_missing: true - rename: + tag: rename_google_workspace_admin_GROUP_MEMBER_BULK_UPLOAD_TOTAL_NUMBER_to_google_workspace_admin_bulk_upload_total_1a0d274c field: google_workspace.admin.GROUP_MEMBER_BULK_UPLOAD_TOTAL_NUMBER target_field: google_workspace.admin.bulk_upload.total ignore_missing: true - rename: + tag: rename_google_workspace_admin_BULK_UPLOAD_FAIL_USERS_NUMBER_to_google_workspace_admin_bulk_upload_failed_ce44c85f field: google_workspace.admin.BULK_UPLOAD_FAIL_USERS_NUMBER target_field: google_workspace.admin.bulk_upload.failed ignore_missing: true - rename: + tag: rename_google_workspace_admin_BULK_UPLOAD_TOTAL_USERS_NUMBER_to_google_workspace_admin_bulk_upload_total_e963afa0 field: google_workspace.admin.BULK_UPLOAD_TOTAL_USERS_NUMBER target_field: google_workspace.admin.bulk_upload.total ignore_missing: true - rename: + tag: rename_google_workspace_admin_EMAIL_LOG_SEARCH_MSG_ID_to_google_workspace_admin_email_log_search_filter_message_id_12147075 field: google_workspace.admin.EMAIL_LOG_SEARCH_MSG_ID target_field: google_workspace.admin.email.log_search_filter.message_id ignore_missing: true - rename: + tag: rename_google_workspace_admin_EMAIL_LOG_SEARCH_RECIPIENT_to_google_workspace_admin_email_log_search_filter_recipient_value_b0664fc2 field: google_workspace.admin.EMAIL_LOG_SEARCH_RECIPIENT target_field: google_workspace.admin.email.log_search_filter.recipient.value ignore_missing: true - rename: + tag: rename_google_workspace_admin_EMAIL_LOG_SEARCH_SENDER_to_google_workspace_admin_email_log_search_filter_sender_value_3ae9afe6 field: google_workspace.admin.EMAIL_LOG_SEARCH_SENDER target_field: google_workspace.admin.email.log_search_filter.sender.value ignore_missing: true - convert: + tag: convert_google_workspace_admin_EMAIL_LOG_SEARCH_SMTP_RECIPIENT_IP_ee5a16e0 field: google_workspace.admin.EMAIL_LOG_SEARCH_SMTP_RECIPIENT_IP type: ip ignore_missing: true on_failure: - remove: + tag: remove_google_workspace_admin_EMAIL_LOG_SEARCH_SMTP_RECIPIENT_IP_6330f121 field: google_workspace.admin.EMAIL_LOG_SEARCH_SMTP_RECIPIENT_IP - rename: + tag: rename_google_workspace_admin_EMAIL_LOG_SEARCH_SMTP_RECIPIENT_IP_to_google_workspace_admin_email_log_search_filter_recipient_ip_3ce73f7f field: google_workspace.admin.EMAIL_LOG_SEARCH_SMTP_RECIPIENT_IP target_field: google_workspace.admin.email.log_search_filter.recipient.ip ignore_missing: true - convert: + tag: convert_google_workspace_admin_EMAIL_LOG_SEARCH_SMTP_SENDER_IP_8eab0eaa field: google_workspace.admin.EMAIL_LOG_SEARCH_SMTP_SENDER_IP type: ip ignore_missing: true on_failure: - remove: + tag: remove_google_workspace_admin_EMAIL_LOG_SEARCH_SMTP_SENDER_IP_5e60cf0f field: google_workspace.admin.EMAIL_LOG_SEARCH_SMTP_SENDER_IP - rename: + tag: rename_google_workspace_admin_EMAIL_LOG_SEARCH_SMTP_SENDER_IP_to_google_workspace_admin_email_log_search_filter_sender_ip_313c4ae3 field: google_workspace.admin.EMAIL_LOG_SEARCH_SMTP_SENDER_IP target_field: google_workspace.admin.email.log_search_filter.sender.ip ignore_missing: true - rename: + tag: rename_google_workspace_admin_QUARANTINE_NAME_to_google_workspace_admin_email_quarantine_name_c8062536 field: google_workspace.admin.QUARANTINE_NAME target_field: google_workspace.admin.email.quarantine_name ignore_missing: true - rename: + tag: rename_google_workspace_admin_CHROME_LICENSES_ENABLED_to_google_workspace_admin_chrome_licenses_enabled_8b237053 field: google_workspace.admin.CHROME_LICENSES_ENABLED target_field: google_workspace.admin.chrome_licenses.enabled ignore_missing: true - rename: + tag: rename_google_workspace_admin_CHROME_LICENSES_ALLOWED_to_google_workspace_admin_chrome_licenses_allowed_e8fc0937 field: google_workspace.admin.CHROME_LICENSES_ALLOWED target_field: google_workspace.admin.chrome_licenses.allowed ignore_missing: true - rename: + tag: rename_google_workspace_admin_FULL_ORG_UNIT_PATH_to_google_workspace_admin_org_unit_full_3981b5e3 field: google_workspace.admin.FULL_ORG_UNIT_PATH target_field: google_workspace.admin.org_unit.full ignore_missing: true - rename: + tag: rename_google_workspace_admin_OAUTH2_SERVICE_NAME_to_google_workspace_admin_oauth2_service_name_89228a50 field: google_workspace.admin.OAUTH2_SERVICE_NAME target_field: google_workspace.admin.oauth2.service.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_OAUTH2_APP_ID_to_google_workspace_admin_oauth2_application_id_ade6a2a9 field: google_workspace.admin.OAUTH2_APP_ID target_field: google_workspace.admin.oauth2.application.id ignore_missing: true - rename: + tag: rename_google_workspace_admin_OAUTH2_APP_NAME_to_google_workspace_admin_oauth2_application_name_a48561e5 field: google_workspace.admin.OAUTH2_APP_NAME target_field: google_workspace.admin.oauth2.application.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_OAUTH2_APP_TYPE_to_google_workspace_admin_oauth2_application_type_8f07b7d9 field: google_workspace.admin.OAUTH2_APP_TYPE target_field: google_workspace.admin.oauth2.application.type ignore_missing: true - rename: + tag: rename_google_workspace_admin_ALLOWED_TWO_STEP_VERIFICATION_METHOD_to_google_workspace_admin_verification_method_19ca231b field: google_workspace.admin.ALLOWED_TWO_STEP_VERIFICATION_METHOD target_field: google_workspace.admin.verification_method ignore_missing: true - rename: + tag: rename_google_workspace_admin_DOMAIN_VERIFICATION_METHOD_to_google_workspace_admin_verification_method_1bccd823 field: google_workspace.admin.DOMAIN_VERIFICATION_METHOD target_field: google_workspace.admin.verification_method ignore_missing: true - rename: + tag: rename_google_workspace_admin_CAA_ASSIGNMENTS_NEW_to_google_workspace_admin_new_value_837bd3b9 field: google_workspace.admin.CAA_ASSIGNMENTS_NEW target_field: google_workspace.admin.new_value ignore_missing: true - rename: + tag: rename_google_workspace_admin_CAA_ASSIGNMENTS_OLD_to_google_workspace_admin_old_value_fb3cf009 field: google_workspace.admin.CAA_ASSIGNMENTS_OLD target_field: google_workspace.admin.old_value ignore_missing: true - rename: + tag: rename_google_workspace_admin_REAUTH_SETTING_NEW_to_google_workspace_admin_new_value_9002e401 field: google_workspace.admin.REAUTH_SETTING_NEW target_field: google_workspace.admin.new_value ignore_missing: true - rename: + tag: rename_google_workspace_admin_REAUTH_SETTING_OLD_to_google_workspace_admin_old_value_672045f1 field: google_workspace.admin.REAUTH_SETTING_OLD target_field: google_workspace.admin.old_value ignore_missing: true @@ -1004,6 +628,7 @@ processors: if: ctx.google_workspace?.admin?.RELATED_ALERT_ID != '' on_failure: - append: + tag: append_error_message_bac90dc1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1047,134 +672,167 @@ processors: target_field: google_workspace.admin.investigation.url_display_text ignore_missing: true - rename: + tag: rename_google_workspace_admin_ALERT_NAME_to_google_workspace_admin_alert_name_7147e31f field: google_workspace.admin.ALERT_NAME target_field: google_workspace.admin.alert.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_API_CLIENT_NAME_to_google_workspace_admin_api_client_name_fa5e4ede field: google_workspace.admin.API_CLIENT_NAME target_field: google_workspace.admin.api.client.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_API_SCOPES_to_google_workspace_admin_api_scopes_db3d5827 field: google_workspace.admin.API_SCOPES target_field: google_workspace.admin.api.scopes ignore_missing: true - rename: + tag: rename_google_workspace_admin_PLAY_FOR_WORK_TOKEN_ID_to_google_workspace_admin_mdm_token_7c3c7283 field: google_workspace.admin.PLAY_FOR_WORK_TOKEN_ID target_field: google_workspace.admin.mdm.token ignore_missing: true - rename: + tag: rename_google_workspace_admin_PLAY_FOR_WORK_MDM_VENDOR_NAME_to_google_workspace_admin_mdm_vendor_ec4150ee field: google_workspace.admin.PLAY_FOR_WORK_MDM_VENDOR_NAME target_field: google_workspace.admin.mdm.vendor ignore_missing: true - rename: + tag: rename_google_workspace_admin_INFO_TYPE_to_google_workspace_admin_info_type_6caf01e8 field: google_workspace.admin.INFO_TYPE target_field: google_workspace.admin.info_type ignore_missing: true - rename: + tag: rename_google_workspace_admin_RULE_NAME_to_google_workspace_admin_rule_name_31b8c065 field: google_workspace.admin.RULE_NAME target_field: google_workspace.admin.rule.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_USER_CUSTOM_FIELD_to_google_workspace_admin_setting_name_190357d3 field: google_workspace.admin.USER_CUSTOM_FIELD target_field: google_workspace.admin.setting.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_EMAIL_MONITOR_DEST_EMAIL_to_google_workspace_admin_email_monitor_dest_email_726f3875 field: google_workspace.admin.EMAIL_MONITOR_DEST_EMAIL target_field: google_workspace.admin.email_monitor.dest_email ignore_missing: true - rename: + tag: rename_google_workspace_admin_EMAIL_MONITOR_LEVEL_CHAT_to_google_workspace_admin_email_monitor_level_chat_149b5d5c field: google_workspace.admin.EMAIL_MONITOR_LEVEL_CHAT target_field: google_workspace.admin.email_monitor.level.chat ignore_missing: true - rename: + tag: rename_google_workspace_admin_EMAIL_MONITOR_LEVEL_DRAFT_EMAIL_to_google_workspace_admin_email_monitor_level_draft_9f71b155 field: google_workspace.admin.EMAIL_MONITOR_LEVEL_DRAFT_EMAIL target_field: google_workspace.admin.email_monitor.level.draft ignore_missing: true - rename: + tag: rename_google_workspace_admin_EMAIL_MONITOR_LEVEL_INCOMING_EMAIL_to_google_workspace_admin_email_monitor_level_incoming_73057fa7 field: google_workspace.admin.EMAIL_MONITOR_LEVEL_INCOMING_EMAIL target_field: google_workspace.admin.email_monitor.level.incoming ignore_missing: true - rename: + tag: rename_google_workspace_admin_EMAIL_MONITOR_LEVEL_OUTGOING_EMAIL_to_google_workspace_admin_email_monitor_level_outgoing_3dbb404f field: google_workspace.admin.EMAIL_MONITOR_LEVEL_OUTGOING_EMAIL target_field: google_workspace.admin.email_monitor.level.outgoing ignore_missing: true - rename: + tag: rename_google_workspace_admin_EMAIL_EXPORT_INCLUDE_DELETED_to_google_workspace_admin_email_dump_include_deleted_a734da29 field: google_workspace.admin.EMAIL_EXPORT_INCLUDE_DELETED target_field: google_workspace.admin.email_dump.include_deleted ignore_missing: true - rename: + tag: rename_google_workspace_admin_EMAIL_EXPORT_PACKAGE_CONTENT_to_google_workspace_admin_email_dump_package_content_b844c829 field: google_workspace.admin.EMAIL_EXPORT_PACKAGE_CONTENT target_field: google_workspace.admin.email_dump.package_content ignore_missing: true - rename: + tag: rename_google_workspace_admin_SEARCH_QUERY_FOR_DUMP_to_google_workspace_admin_email_dump_query_d66935d9 field: google_workspace.admin.SEARCH_QUERY_FOR_DUMP target_field: google_workspace.admin.email_dump.query ignore_missing: true - rename: + tag: rename_google_workspace_admin_DESTINATION_USER_EMAIL_to_google_workspace_admin_new_value_43f08219 field: google_workspace.admin.DESTINATION_USER_EMAIL target_field: google_workspace.admin.new_value ignore_missing: true - rename: + tag: rename_google_workspace_admin_REQUEST_ID_to_google_workspace_admin_request_id_9817afc5 field: google_workspace.admin.REQUEST_ID target_field: google_workspace.admin.request.id ignore_missing: true - rename: + tag: rename_google_workspace_admin_GMAIL_RESET_REASON_to_message_e950d4ce field: google_workspace.admin.GMAIL_RESET_REASON target_field: message ignore_missing: true - rename: + tag: rename_google_workspace_admin_USER_NICKNAME_to_google_workspace_admin_user_nickname_5a7a65e9 field: google_workspace.admin.USER_NICKNAME target_field: google_workspace.admin.user.nickname ignore_missing: true - rename: + tag: rename_google_workspace_admin_ACTION_ID_to_google_workspace_admin_mobile_action_id_0b99088d field: google_workspace.admin.ACTION_ID target_field: google_workspace.admin.mobile.action.id ignore_missing: true - rename: + tag: rename_google_workspace_admin_ACTION_TYPE_to_google_workspace_admin_mobile_action_type_82a58f2f field: google_workspace.admin.ACTION_TYPE target_field: google_workspace.admin.mobile.action.type ignore_missing: true - rename: + tag: rename_google_workspace_admin_MOBILE_CERTIFICATE_COMMON_NAME_to_google_workspace_admin_mobile_certificate_name_f892031e field: google_workspace.admin.MOBILE_CERTIFICATE_COMMON_NAME target_field: google_workspace.admin.mobile.certificate.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_NUMBER_OF_COMPANY_OWNED_DEVICES_to_google_workspace_admin_mobile_company_owned_devices_458dfbaa field: google_workspace.admin.NUMBER_OF_COMPANY_OWNED_DEVICES target_field: google_workspace.admin.mobile.company_owned_devices ignore_missing: true - rename: + tag: rename_google_workspace_admin_COMPANY_DEVICE_ID_to_google_workspace_admin_device_id_bf913acd field: google_workspace.admin.COMPANY_DEVICE_ID target_field: google_workspace.admin.device.id ignore_missing: true - rename: + tag: rename_google_workspace_admin_DISTRIBUTION_ENTITY_NAME_to_google_workspace_admin_distribution_entity_name_45f71920 field: google_workspace.admin.DISTRIBUTION_ENTITY_NAME target_field: google_workspace.admin.distribution.entity.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_DISTRIBUTION_ENTITY_TYPE_to_google_workspace_admin_distribution_entity_type_9e479386 field: google_workspace.admin.DISTRIBUTION_ENTITY_TYPE target_field: google_workspace.admin.distribution.entity.type ignore_missing: true - rename: + tag: rename_google_workspace_admin_MOBILE_APP_PACKAGE_ID_to_google_workspace_admin_application_package_id_a3501123 field: google_workspace.admin.MOBILE_APP_PACKAGE_ID target_field: google_workspace.admin.application.package_id ignore_missing: true - rename: + tag: rename_google_workspace_admin_NEW_PERMISSION_GRANT_STATE_to_google_workspace_admin_new_value_a686cf77 field: google_workspace.admin.NEW_PERMISSION_GRANT_STATE target_field: google_workspace.admin.new_value ignore_missing: true - rename: + tag: rename_google_workspace_admin_OLD_PERMISSION_GRANT_STATE_to_google_workspace_admin_old_value_e278d889 field: google_workspace.admin.OLD_PERMISSION_GRANT_STATE target_field: google_workspace.admin.old_value ignore_missing: true - rename: + tag: rename_google_workspace_admin_PERMISSION_GROUP_NAME_to_google_workspace_admin_setting_name_b29fdce0 field: google_workspace.admin.PERMISSION_GROUP_NAME target_field: google_workspace.admin.setting.name ignore_missing: true - rename: + tag: rename_google_workspace_admin_MOBILE_WIRELESS_NETWORK_NAME_to_network_name_fd8bb855 field: google_workspace.admin.MOBILE_WIRELESS_NETWORK_NAME target_field: network.name ignore_missing: true - date: + tag: date_google_workspace_admin_EMAIL_LOG_SEARCH_END_DATE_to_google_workspace_admin_email_log_search_filter_end_date_426fbd79 field: google_workspace.admin.EMAIL_LOG_SEARCH_END_DATE target_field: google_workspace.admin.email.log_search_filter.end_date timezone: UTC @@ -1187,6 +845,7 @@ processors: - yyyy/MM/dd HH:mm z if: ctx?.google_workspace?.admin?.EMAIL_LOG_SEARCH_END_DATE != null - date: + tag: date_google_workspace_admin_EMAIL_LOG_SEARCH_START_DATE_to_google_workspace_admin_email_log_search_filter_start_date_77024bb6 field: google_workspace.admin.EMAIL_LOG_SEARCH_START_DATE target_field: google_workspace.admin.email.log_search_filter.start_date timezone: UTC @@ -1199,6 +858,7 @@ processors: - yyyy/MM/dd HH:mm z if: ctx?.google_workspace?.admin?.EMAIL_LOG_SEARCH_START_DATE != null - date: + tag: date_google_workspace_admin_BIRTHDATE_to_google_workspace_admin_user_birthdate_e2edae21 field: google_workspace.admin.BIRTHDATE target_field: google_workspace.admin.user.birthdate timezone: UTC @@ -1211,6 +871,7 @@ processors: - yyyy/MM/dd HH:mm z if: ctx?.google_workspace?.admin?.BIRTHDATE != null - date: + tag: date_google_workspace_admin_BEGIN_DATE_TIME_to_event_start_66b984e9 field: google_workspace.admin.BEGIN_DATE_TIME target_field: event.start timezone: UTC @@ -1223,6 +884,7 @@ processors: - yyyy/MM/dd HH:mm z if: ctx?.google_workspace?.admin?.BEGIN_DATE_TIME != null - date: + tag: date_google_workspace_admin_START_DATE_to_event_start_d38471ab field: google_workspace.admin.START_DATE target_field: event.start timezone: UTC @@ -1235,6 +897,7 @@ processors: - yyyy/MM/dd HH:mm z if: ctx?.google_workspace?.admin?.START_DATE != null - date: + tag: date_google_workspace_admin_END_DATE_to_event_end_bd30f6a0 field: google_workspace.admin.END_DATE target_field: event.end timezone: UTC @@ -1247,6 +910,7 @@ processors: - yyyy/MM/dd HH:mm z if: ctx?.google_workspace?.admin?.END_DATE != null - date: + tag: date_google_workspace_admin_END_DATE_TIME_to_event_end_cc698494 field: google_workspace.admin.END_DATE_TIME target_field: event.end timezone: UTC @@ -1259,6 +923,7 @@ processors: - yyyy/MM/dd HH:mm z if: ctx?.google_workspace?.admin?.END_DATE_TIME != null - script: + tag: script_8d756e1a lang: painless if: 'ctx?.source?.user?.email != null && ctx?.source?.user?.email.contains("@")' source: > @@ -1274,6 +939,7 @@ processors: ctx.user.domain = splitmail[1]; ctx.source.user.domain = splitmail[1]; - script: + tag: script_50977d57 lang: painless if: 'ctx?.google_workspace?.admin?.group?.email != null && ctx?.google_workspace?.admin?.group?.email.contains("@")' source: > @@ -1287,6 +953,7 @@ processors: ctx.group.name = splitmail[0]; ctx.group.domain = splitmail[1]; - script: + tag: script_ac4be34c lang: painless if: 'ctx?.google_workspace?.admin?.user?.email != null && ctx?.google_workspace?.admin?.user?.email.contains("@")' source: > @@ -1307,14 +974,17 @@ processors: ctx.user.target.domain = splitmail[1]; ctx.user.target.email = ctx.google_workspace.admin.user.email; - set: + tag: set_user_target_group_name_fa952c70 field: user.target.group.name copy_from: group.name if: ctx?.group?.name != null - set: + tag: set_user_target_group_domain_09d0d95f field: user.target.group.domain copy_from: group.domain if: ctx?.group?.domain != null - script: + tag: script_ca699a4b lang: painless if: 'ctx?.event?.start != null && ctx?.event?.end != null' source: >- @@ -1322,46 +992,56 @@ processors: ZonedDateTime end = ZonedDateTime.parse(ctx.event.end); ctx.event.duration = ChronoUnit.NANOS.between(start, end); - convert: + tag: convert_google_workspace_admin_bulk_upload_total_7b888139 field: google_workspace.admin.bulk_upload.total type: long ignore_missing: true - convert: + tag: convert_google_workspace_admin_bulk_upload_failed_6cfe212c field: google_workspace.admin.bulk_upload.failed type: long ignore_missing: true - set: + tag: set_event_outcome_3df3c34d field: event.outcome value: success if: 'ctx?.google_workspace?.admin?.group?.bulk_upload?.failed != null && ctx?.google_workspace?.admin?.group?.bulk_upload?.failed == 0' - set: + tag: set_event_outcome_924198ca field: event.outcome value: failure if: 'ctx?.google_workspace?.admin?.group?.bulk_upload?.failed != null && ctx?.google_workspace?.admin?.group?.bulk_upload?.failed != 0' - split: + tag: split_google_workspace_admin_WHITELISTED_GROUPS_to_google_workspace_admin_group_allowed_list_e16e9fbe field: google_workspace.admin.WHITELISTED_GROUPS target_field: google_workspace.admin.group.allowed_list separator: ',' ignore_missing: true - append: + tag: append_related_ip_549fa858 field: related.ip value: "{{{source.ip}}}" if: ctx?.source?.ip != null allow_duplicates: false - append: + tag: append_related_user_8302ba1f field: related.user value: "{{{source.user.name}}}" if: ctx?.source?.user?.name != null allow_duplicates: false - append: + tag: append_related_user_5fca28ff field: related.user value: "{{{user.target.name}}}" if: ctx?.user?.target?.name != null allow_duplicates: false - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -1370,26 +1050,32 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - convert: + tag: convert_event_id_d2cffc63 field: event.id type: string ignore_missing: true - convert: + tag: convert_source_user_id_bdef6921 field: source.user.id type: string ignore_missing: true - convert: + tag: convert_user_id_c132c028 field: user.id type: string ignore_missing: true - remove: + tag: remove_260f707c field: - json - google_workspace.admin.EMAIL_LOG_SEARCH_END_DATE diff --git a/packages/google_workspace/data_stream/alert/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/alert/elasticsearch/ingest_pipeline/default.yml index 3e6cb2360de..db8f8d669f3 100644 --- a/packages/google_workspace/data_stream/alert/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/alert/elasticsearch/ingest_pipeline/default.yml @@ -13,28 +13,35 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - set: + tag: set_ecs_version_135371bc field: ecs.version value: '8.16.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - append: + tag: append_event_type_8a66ccaa field: event.type value: info - set: + tag: set_event_kind_39295792 field: event.kind value: alert - append: + tag: append_event_category_5b8d356a field: event.category if: ctx.json.source == 'Gmail phishing' value: @@ -42,12 +49,14 @@ processors: - threat - malware - append: + tag: append_event_category_8edb1962 field: event.category if: ctx.json.source != 'Gmail phishing' value: - threat - malware - fingerprint: + tag: fingerprint_059823db fields: - json.alertId - json.customerId @@ -58,8 +67,9 @@ processors: target_field: _id ignore_missing: true -# Pipeline For Common fields. + # Pipeline For Common fields. - date: + tag: date_json_createTime_to_timestamp_68ca4c26 field: json.createTime target_field: '@timestamp' if: ctx.json?.createTime != null @@ -72,13 +82,16 @@ processors: - yyyy/MM/dd HH:mm:ss z on_failure: - append: + tag: append_error_message_52b8cfb3 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_alert_create_time_a919a523 field: google_workspace.alert.create_time copy_from: '@timestamp' ignore_failure: true - date: + tag: date_json_endTime_to_google_workspace_alert_end_time_e1cc56ca field: json.endTime if: ctx.json?.endTime != null timezone: UTC @@ -91,13 +104,16 @@ processors: target_field: google_workspace.alert.end_time on_failure: - append: + tag: append_error_message_ca8f8ec1 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_event_end_feeb2689 field: event.end copy_from: google_workspace.alert.end_time ignore_failure: true - date: + tag: date_json_startTime_to_google_workspace_alert_start_time_ca58c857 field: json.startTime if: ctx.json?.startTime != null timezone: UTC @@ -110,84 +126,104 @@ processors: target_field: google_workspace.alert.start_time on_failure: - append: + tag: append_error_message_16e621b0 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_event_start_3e0ba55f field: event.start copy_from: google_workspace.alert.start_time ignore_failure: true - rename: + tag: rename_json_source_to_google_workspace_alert_source_0078390f field: json.source target_field: google_workspace.alert.source ignore_missing: true - set: + tag: set_event_action_9c5f46ec field: event.action copy_from: google_workspace.alert.source ignore_failure: true - rename: + tag: rename_json_customerId_to_google_workspace_alert_customer_id_f17f73f5 field: json.customerId target_field: google_workspace.alert.customer.id ignore_missing: true - set: + tag: set_organization_id_3a5c2576 field: organization.id copy_from: google_workspace.alert.customer.id ignore_failure: true - rename: + tag: rename_json_metadata_assignee_to_google_workspace_alert_metadata_assignee_48f4dec1 field: json.metadata.assignee target_field: google_workspace.alert.metadata.assignee ignore_missing: true - append: + tag: append_user_email_30a34aeb field: user.email value: '{{{google_workspace.alert.metadata.assignee}}}' if: ctx.google_workspace?.alert?.metadata?.assignee != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_alertId_to_google_workspace_alert_id_08028acb field: json.alertId target_field: google_workspace.alert.id ignore_missing: true - set: + tag: set_event_id_71fa2dd5 field: event.id copy_from: google_workspace.alert.id ignore_failure: true - convert: + tag: convert_json_deleted_to_google_workspace_alert_deleted_930fb5eb field: json.deleted target_field: google_workspace.alert.deleted type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_8baa31be field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_etag_to_google_workspace_alert_etag_e9cc9c63 field: json.etag target_field: google_workspace.alert.etag ignore_missing: true - rename: + tag: rename_json_metadata_alertId_to_google_workspace_alert_metadata_alert_id_6834fa0b field: json.metadata.alertId target_field: google_workspace.alert.metadata.alert.id ignore_missing: true - rename: + tag: rename_json_metadata_customerId_to_google_workspace_alert_metadata_customer_id_bc0421fb field: json.metadata.customerId target_field: google_workspace.alert.metadata.customer.id ignore_missing: true - rename: + tag: rename_json_data_type_to_google_workspace_alert_data_type_5f678f23 field: json.data.@type target_field: google_workspace.alert.data.type ignore_missing: true - rename: + tag: rename_json_metadata_etag_to_google_workspace_alert_metadata_etag_0914a475 field: json.metadata.etag target_field: google_workspace.alert.metadata.etag ignore_missing: true - rename: + tag: rename_json_metadata_severity_to_google_workspace_alert_metadata_severity_cfa6c4c1 field: json.metadata.severity target_field: google_workspace.alert.metadata.severity ignore_missing: true - rename: + tag: rename_json_metadata_status_to_google_workspace_alert_metadata_status_4a095d59 field: json.metadata.status target_field: google_workspace.alert.metadata.status ignore_missing: true - date: + tag: date_json_metadata_updateTime_to_google_workspace_alert_metadata_update_time_c371e716 field: json.metadata.updateTime if: ctx.json?.metadata?.updateTime != null timezone: UTC @@ -200,17 +236,21 @@ processors: target_field: google_workspace.alert.metadata.update_time on_failure: - append: + tag: append_error_message_98ade8c7 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_securityInvestigationToolLink_to_google_workspace_alert_security_investigation_tool_link_c8760b00 field: json.securityInvestigationToolLink target_field: google_workspace.alert.security_investigation_tool_link ignore_missing: true - rename: + tag: rename_json_type_to_google_workspace_alert_type_24d35ea9 field: json.type target_field: google_workspace.alert.type ignore_missing: true - date: + tag: date_json_updateTime_to_google_workspace_alert_update_time_33a29d2a field: json.updateTime if: ctx.json?.updateTime != null timezone: UTC @@ -223,31 +263,37 @@ processors: target_field: google_workspace.alert.update_time on_failure: - append: + tag: append_error_message_6329016b field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_data_email_to_google_workspace_alert_data_email_832cfd05 field: json.data.email target_field: google_workspace.alert.data.email ignore_missing: true - append: + tag: append_user_email_954c1f73 field: user.email value: '{{{google_workspace.alert.data.email}}}' if: ctx.google_workspace?.alert?.data?.email != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_data_alertDetails_to_google_workspace_alert_data_alert_details_5d4df09a field: json.data.alertDetails target_field: google_workspace.alert.data.alert_details ignore_missing: true -# Pipeline For Domain wide takeout. + # Pipeline For Domain wide takeout. - rename: + tag: rename_json_data_takeoutRequestId_to_google_workspace_alert_data_takeout_request_id_2a283cbb field: json.data.takeoutRequestId target_field: google_workspace.alert.data.takeout.request.id ignore_missing: true -# Pipeline For Gmail phishing. + # Pipeline For Gmail phishing. - foreach: + tag: foreach_json_data_messages_1172116e field: json.data.messages processor: rename: @@ -257,6 +303,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_messages_6b6b3956 field: json.data.messages processor: date: @@ -273,6 +320,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_messages_2c4cf9e5 field: json.data.messages processor: append: @@ -282,6 +330,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_messages_07a2f116 field: json.data.messages processor: rename: @@ -291,6 +340,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_messages_3ca60ccc field: json.data.messages processor: rename: @@ -300,6 +350,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_messages_c869b9ab field: json.data.messages processor: rename: @@ -309,6 +360,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_messages_6beed4bb field: json.data.messages processor: rename: @@ -318,6 +370,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_messages_3db57e4a field: json.data.messages processor: append: @@ -327,6 +380,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_messages_031bdc3f field: json.data.messages processor: rename: @@ -336,6 +390,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_messages_721621cd field: json.data.messages processor: append: @@ -345,6 +400,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_messages_2f6658df field: json.data.messages processor: rename: @@ -354,6 +410,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_messages_25ba9426 field: json.data.messages processor: append: @@ -363,10 +420,12 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_data_messages_to_google_workspace_alert_data_messages_973ab021 field: json.data.messages target_field: google_workspace.alert.data.messages ignore_missing: true - foreach: + tag: foreach_google_workspace_alert_data_messages_282d1899 field: google_workspace.alert.data.messages processor: foreach: @@ -382,6 +441,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_google_workspace_alert_data_messages_54e2b458 field: google_workspace.alert.data.messages processor: append: @@ -391,66 +451,81 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_data_maliciousEntity_entity_emailAddress_to_google_workspace_alert_data_malicious_entity_entity_email_address_2b4bd759 field: json.data.maliciousEntity.entity.emailAddress target_field: google_workspace.alert.data.malicious_entity.entity.email_address ignore_missing: true - append: + tag: append_user_email_4ca189e5 field: user.email value: '{{{google_workspace.alert.data.malicious_entity.entity.email_address}}}' if: ctx.google_workspace?.alert?.data?.malicious_entity?.entity?.email_address != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_data_maliciousEntity_entity_displayName_to_google_workspace_alert_data_malicious_entity_entity_display_name_fcb5bec9 field: json.data.maliciousEntity.entity.displayName target_field: google_workspace.alert.data.malicious_entity.entity.display_name ignore_missing: true - set: + tag: set_user_name_b26192bc field: user.name copy_from: google_workspace.alert.data.malicious_entity.entity.display_name ignore_failure: true - rename: + tag: rename_json_data_domainId_customerPrimaryDomain_to_google_workspace_alert_data_domain_id_customer_primary_domain_300c729e field: json.data.domainId.customerPrimaryDomain target_field: google_workspace.alert.data.domain_id.customer_primary_domain ignore_missing: true - set: + tag: set_user_domain_a13f7672 field: user.domain copy_from: google_workspace.alert.data.domain_id.customer_primary_domain ignore_failure: true - rename: + tag: rename_json_data_maliciousEntity_displayName_to_google_workspace_alert_data_malicious_entity_display_name_39e58a11 field: json.data.maliciousEntity.displayName target_field: google_workspace.alert.data.malicious_entity.display_name ignore_missing: true - rename: + tag: rename_json_data_maliciousEntity_fromHeader_to_google_workspace_alert_data_malicious_entity_from_header_40fbfa03 field: json.data.maliciousEntity.fromHeader target_field: google_workspace.alert.data.malicious_entity.from_header ignore_missing: true - rename: + tag: rename_json_data_systemActionType_to_google_workspace_alert_data_system_action_type_4667b77b field: json.data.systemActionType target_field: google_workspace.alert.data.system_action_type ignore_missing: true - convert: + tag: convert_json_data_isInternal_to_google_workspace_alert_data_is_internal_55d70d12 field: json.data.isInternal target_field: google_workspace.alert.data.is_internal type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_c2805075 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_data_sourceIp_to_google_workspace_alert_data_source_ip_c3c02b0a field: json.data.sourceIp target_field: google_workspace.alert.data.source.ip type: ip ignore_missing: true on_failure: - append: + tag: append_error_message_3fc24edf field: error.message value: '{{{_ingest.on_failure_message}}}' - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -459,29 +534,35 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - set: + tag: set_source_ip_461a1df7 field: source.ip copy_from: google_workspace.alert.data.source.ip ignore_failure: true -# Pipeline For Google identity. + # Pipeline For Google identity. - convert: + tag: convert_json_data_loginDetails_ipAddress_to_google_workspace_alert_data_login_details_ip_address_be6d1924 field: json.data.loginDetails.ipAddress target_field: google_workspace.alert.data.login_details.ip_address type: ip ignore_missing: true on_failure: - append: + tag: append_error_message_606a0247 field: error.message value: '{{{_ingest.on_failure_message}}}' - date: + tag: date_json_data_loginDetails_loginTime_to_google_workspace_alert_data_login_details_login_time_1c65d1ee field: json.data.loginDetails.loginTime if: ctx.json?.data?.loginDetails?.loginTime != null timezone: UTC @@ -494,17 +575,21 @@ processors: target_field: google_workspace.alert.data.login_details.login_time on_failure: - append: + tag: append_error_message_efbb9d5b field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_data_state_to_google_workspace_alert_data_state_46ba5ecf field: json.data.state target_field: google_workspace.alert.data.state ignore_missing: true - rename: + tag: rename_json_data_appealWindow_to_google_workspace_alert_data_appeal_window_d5386514 field: json.data.appealWindow target_field: google_workspace.alert.data.appeal_window ignore_missing: true - foreach: + tag: foreach_json_data_suspensionDetails_01c12526 field: json.data.suspensionDetails processor: rename: @@ -514,6 +599,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_suspensionDetails_c12e7f74 field: json.data.suspensionDetails processor: rename: @@ -523,16 +609,19 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_data_suspensionDetails_to_google_workspace_alert_data_suspension_details_902ddf48 field: json.data.suspensionDetails target_field: google_workspace.alert.data.suspension_details ignore_missing: true -# Pipeline For Google Operations. + # Pipeline For Google Operations. - rename: + tag: rename_json_data_affectedUserEmails_to_google_workspace_alert_data_affected_user_emails_dd2772f2 field: json.data.affectedUserEmails target_field: google_workspace.alert.data.affected.user_emails ignore_missing: true - foreach: + tag: foreach_google_workspace_alert_data_affected_user_emails_73abf0c4 field: google_workspace.alert.data.affected.user_emails processor: append: @@ -543,37 +632,45 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_data_title_to_google_workspace_alert_data_title_d6f1eb61 field: json.data.title target_field: google_workspace.alert.data.title ignore_missing: true - rename: + tag: rename_json_data_description_to_google_workspace_alert_data_description_b2764b7d field: json.data.description if: ctx.event?.action == 'Google Operations' target_field: google_workspace.alert.data.description ignore_missing: true - rename: + tag: rename_json_data_attachmentData_csv_headers_to_google_workspace_alert_data_attachment_data_csv_headers_606ca091 field: json.data.attachmentData.csv.headers target_field: google_workspace.alert.data.attachment.data.csv.headers ignore_missing: true - rename: + tag: rename_json_data_attachmentData_csv_dataRows_to_google_workspace_alert_data_attachment_data_csv_data_rows_33e6eae0 field: json.data.attachmentData.csv.dataRows target_field: google_workspace.alert.data.attachment.data.csv.data_rows ignore_missing: true - rename: + tag: rename_json_data_header_to_google_workspace_alert_data_header_225ac42d field: json.data.header target_field: google_workspace.alert.data.header ignore_missing: true - rename: + tag: rename_json_data_domain_to_google_workspace_alert_data_domain_6f0b8109 field: json.data.domain target_field: google_workspace.alert.data.domain ignore_missing: true - set: + tag: set_user_domain_744d269e field: user.domain copy_from: google_workspace.alert.data.domain ignore_failure: true -# Pipeline For Mobile device management. + # Pipeline For Mobile device management. - foreach: + tag: foreach_json_data_events_58e3d601 field: json.data.events processor: rename: @@ -583,6 +680,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_events_08acc0ed field: json.data.events processor: rename: @@ -592,6 +690,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_events_1cef9629 field: json.data.events processor: rename: @@ -601,6 +700,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_events_77e7a3df field: json.data.events processor: rename: @@ -610,6 +710,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_events_4166a4b1 field: json.data.events processor: rename: @@ -619,6 +720,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_events_6dd63516 field: json.data.events processor: rename: @@ -628,6 +730,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_events_aa769fef field: json.data.events processor: rename: @@ -637,6 +740,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_events_aa11bc7d field: json.data.events processor: rename: @@ -646,6 +750,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_events_31557bbc field: json.data.events processor: rename: @@ -655,6 +760,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_events_1cd2a4ea field: json.data.events processor: rename: @@ -664,12 +770,14 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_data_events_to_google_workspace_alert_data_events_dc116d75 field: json.data.events target_field: google_workspace.alert.data.events ignore_missing: true -# Pipeline For AppMaker Editor. + # Pipeline For AppMaker Editor. - foreach: + tag: foreach_json_data_requestInfo_d1171305 field: json.data.requestInfo processor: rename: @@ -679,6 +787,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_requestInfo_8fbccdf0 field: json.data.requestInfo processor: rename: @@ -688,6 +797,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_requestInfo_99e01c47 field: json.data.requestInfo processor: rename: @@ -697,41 +807,50 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_data_requestInfo_to_google_workspace_alert_data_request_info_e3d30b4b field: json.data.requestInfo target_field: google_workspace.alert.data.request.info ignore_missing: true -# Pipeline For Security Center rules. + # Pipeline For Security Center rules. - rename: + tag: rename_json_data_description_to_google_workspace_alert_data_rule_description_458ef8e3 field: json.data.description if: ctx.event?.action == 'Security Center rules' target_field: google_workspace.alert.data.rule_description ignore_missing: true - set: + tag: set_rule_description_b4265eba field: rule.description copy_from: google_workspace.alert.data.rule_description ignore_failure: true - rename: + tag: rename_json_data_name_to_google_workspace_alert_data_name_42b605a1 field: json.data.name target_field: google_workspace.alert.data.name ignore_missing: true - set: + tag: set_rule_name_dc26a0cb field: rule.name copy_from: google_workspace.alert.data.name ignore_failure: true - rename: + tag: rename_json_data_displayName_to_google_workspace_alert_data_display_name_8fb38481 field: json.data.displayName target_field: google_workspace.alert.data.display.name ignore_missing: true - rename: + tag: rename_json_data_windowSize_to_google_workspace_alert_data_window_size_4e9e2e9e field: json.data.windowSize target_field: google_workspace.alert.data.window_size ignore_missing: true - rename: + tag: rename_json_data_threshold_to_google_workspace_alert_data_threshold_ed381a03 field: json.data.threshold target_field: google_workspace.alert.data.threshold ignore_missing: true - date: + tag: date_json_data_createTime_to_google_workspace_alert_data_create_time_253e3792 field: json.data.createTime if: ctx.json?.data?.createTime != null timezone: UTC @@ -744,9 +863,11 @@ processors: target_field: google_workspace.alert.data.create_time on_failure: - append: + tag: append_error_message_5729422f field: error.message value: '{{{_ingest.on_failure_message}}}' - date: + tag: date_json_data_updateTime_to_google_workspace_alert_data_update_time_e89beec5 field: json.data.updateTime if: ctx.json?.data?.updateTime != null timezone: UTC @@ -759,67 +880,83 @@ processors: target_field: google_workspace.alert.data.update_time on_failure: - append: + tag: append_error_message_7a3b0e08 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_data_triggerSource_to_google_workspace_alert_data_trigger_source_81415dc1 field: json.data.triggerSource target_field: google_workspace.alert.data.trigger.source ignore_missing: true - rename: + tag: rename_json_data_supersededAlerts_to_google_workspace_alert_data_superseded_alerts_c85f6cb6 field: json.data.supersededAlerts target_field: google_workspace.alert.data.superseded_alerts ignore_missing: true - rename: + tag: rename_json_data_supersedingAlert_to_google_workspace_alert_data_superseding_alert_2b0df986 field: json.data.supersedingAlert target_field: google_workspace.alert.data.superseding_alert ignore_missing: true - rename: + tag: rename_json_data_actionNames_to_google_workspace_alert_data_action_name_7398169e field: json.data.actionNames target_field: google_workspace.alert.data.action.name ignore_missing: true - rename: + tag: rename_json_data_query_to_google_workspace_alert_data_query_2344d549 field: json.data.query target_field: google_workspace.alert.data.query ignore_missing: true -# Pipeline For Data Loss Prevention. + # Pipeline For Data Loss Prevention. - rename: + tag: rename_json_data_ruleViolationInfo_ruleInfo_displayName_to_google_workspace_alert_data_rule_violation_info_rule_info_display_name_0fe2f603 field: json.data.ruleViolationInfo.ruleInfo.displayName target_field: google_workspace.alert.data.rule.violation_info.rule_info.display.name ignore_missing: true - set: + tag: set_rule_name_673bf10c field: rule.name copy_from: google_workspace.alert.data.rule.violation_info.rule_info.display.name ignore_failure: true - rename: + tag: rename_json_data_ruleViolationInfo_ruleInfo_resourceName_to_google_workspace_alert_data_rule_violation_info_rule_info_resource_name_79dcbdb7 field: json.data.ruleViolationInfo.ruleInfo.resourceName target_field: google_workspace.alert.data.rule.violation_info.rule_info.resource.name ignore_missing: true - rename: + tag: rename_json_data_ruleViolationInfo_dataSource_to_google_workspace_alert_data_rule_violation_info_data_source_195db44c field: json.data.ruleViolationInfo.dataSource target_field: google_workspace.alert.data.rule.violation_info.data.source ignore_missing: true - rename: + tag: rename_json_data_ruleViolationInfo_trigger_to_google_workspace_alert_data_rule_violation_info_trigger_value_b0c06bf5 field: json.data.ruleViolationInfo.trigger target_field: google_workspace.alert.data.rule.violation_info.trigger.value ignore_missing: true - rename: + tag: rename_json_data_ruleViolationInfo_triggeringUserEmail_to_google_workspace_alert_data_rule_violation_info_trigger_user_email_d12ad2f6 field: json.data.ruleViolationInfo.triggeringUserEmail target_field: google_workspace.alert.data.rule.violation_info.trigger.user.email ignore_missing: true - rename: + tag: rename_json_data_ruleViolationInfo_recipients_to_google_workspace_alert_data_rule_violation_info_recipients_c4bcaec0 field: json.data.ruleViolationInfo.recipients target_field: google_workspace.alert.data.rule.violation_info.recipients ignore_missing: true - rename: + tag: rename_json_data_ruleViolationInfo_resourceInfo_resourceTitle_to_google_workspace_alert_data_rule_violation_info_resource_info_resource_title_82657c97 field: json.data.ruleViolationInfo.resourceInfo.resourceTitle target_field: google_workspace.alert.data.rule.violation_info.resource_info.resource.title ignore_missing: true - rename: + tag: rename_json_data_ruleViolationInfo_resourceInfo_documentId_to_google_workspace_alert_data_rule_violation_info_resource_info_document_id_2ebb3fcb field: json.data.ruleViolationInfo.resourceInfo.documentId target_field: google_workspace.alert.data.rule.violation_info.resource_info.document.id ignore_missing: true - foreach: + tag: foreach_json_data_ruleViolationInfo_matchInfo_0ee4d162 field: json.data.ruleViolationInfo.matchInfo processor: rename: @@ -829,6 +966,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_ruleViolationInfo_matchInfo_59d3619c field: json.data.ruleViolationInfo.matchInfo processor: rename: @@ -838,6 +976,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_data_ruleViolationInfo_matchInfo_e9c86dd3 field: json.data.ruleViolationInfo.matchInfo processor: rename: @@ -847,28 +986,34 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_data_ruleViolationInfo_matchInfo_to_google_workspace_alert_data_rule_violation_info_match_info_ab727ced field: json.data.ruleViolationInfo.matchInfo target_field: google_workspace.alert.data.rule.violation_info.match_info ignore_missing: true - rename: + tag: rename_json_data_ruleViolationInfo_triggeredActionTypes_to_google_workspace_alert_data_rule_violation_info_triggered_action_types_1b9ab6b8 field: json.data.ruleViolationInfo.triggeredActionTypes target_field: google_workspace.alert.data.rule.violation_info.triggered.action.types ignore_missing: true - rename: + tag: rename_json_data_ruleViolationInfo_triggeredActionInfo_to_google_workspace_alert_data_rule_violation_info_triggered_action_info_d1bb5542 field: json.data.ruleViolationInfo.triggeredActionInfo target_field: google_workspace.alert.data.rule.violation_info.triggered.action.info ignore_missing: true - rename: + tag: rename_json_data_ruleViolationInfo_suppressedActionTypes_to_google_workspace_alert_data_rule_violation_info_suppressed_action_types_80d5b5fa field: json.data.ruleViolationInfo.suppressedActionTypes target_field: google_workspace.alert.data.rule.violation_info.suppressed.action.types ignore_missing: true -# Pipeline For Apps outage. + # Pipeline For Apps outage. - rename: + tag: rename_json_data_products_to_google_workspace_alert_data_products_d0662f81 field: json.data.products target_field: google_workspace.alert.data.products ignore_missing: true - date: + tag: date_json_data_nextUpdateTime_to_google_workspace_alert_data_next_update_time_fc93fd8d field: json.data.nextUpdateTime if: ctx.json?.data?.nextUpdateTime != null timezone: UTC @@ -881,9 +1026,11 @@ processors: target_field: google_workspace.alert.data.next_update_time on_failure: - append: + tag: append_error_message_bd53de8e field: error.message value: '{{{_ingest.on_failure_message}}}' - date: + tag: date_json_data_resolutionTime_to_google_workspace_alert_data_resolution_time_272a3c40 field: json.data.resolutionTime if: ctx.json?.data?.resolutionTime != null timezone: UTC @@ -896,39 +1043,48 @@ processors: target_field: google_workspace.alert.data.resolution_time on_failure: - append: + tag: append_error_message_5b0c0b35 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_data_dashboardUri_to_google_workspace_alert_data_dashboard_uri_33a05b41 field: json.data.dashboardUri target_field: google_workspace.alert.data.dashboard.uri ignore_missing: true - rename: + tag: rename_json_data_status_to_google_workspace_alert_data_status_69832261 field: json.data.status target_field: google_workspace.alert.data.status ignore_missing: true - rename: + tag: rename_json_data_incidentTrackingId_to_google_workspace_alert_data_incident_tracking_id_fc74afd4 field: json.data.incidentTrackingId target_field: google_workspace.alert.data.incident_tracking.id ignore_missing: true - rename: + tag: rename_json_data_mergeInfo_newIncidentTrackingId_to_google_workspace_alert_data_merge_info_new_incident_tracking_id_96d50242 field: json.data.mergeInfo.newIncidentTrackingId target_field: google_workspace.alert.data.merge_info.new_incident_tracking.id ignore_missing: true - rename: + tag: rename_json_data_mergeInfo_newAlertId_to_google_workspace_alert_data_merge_info_new_alert_id_79258045 field: json.data.mergeInfo.newAlertId target_field: google_workspace.alert.data.merge_info.new_alert.id ignore_missing: true -# Pipeline For Sensitive Admin Action. + # Pipeline For Sensitive Admin Action. - rename: + tag: rename_json_data_actorEmail_to_google_workspace_alert_data_actor_email_295c335d field: json.data.actorEmail target_field: google_workspace.alert.data.actor.email ignore_missing: true - set: + tag: set_source_user_email_5067cc52 field: source.user.email copy_from: google_workspace.alert.data.actor.email ignore_failure: true - date: + tag: date_json_data_eventTime_to_google_workspace_alert_data_event_time_137cb4f0 field: json.data.eventTime if: ctx.json?.data?.eventTime != null timezone: UTC @@ -941,41 +1097,51 @@ processors: target_field: google_workspace.alert.data.event_time on_failure: - append: + tag: append_error_message_cc39d383 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_data_primaryAdminChangedEvent_domain_to_google_workspace_alert_data_primary_admin_changed_event_domain_33c1cb64 field: json.data.primaryAdminChangedEvent.domain target_field: google_workspace.alert.data.primary.admin.changed_event.domain ignore_missing: true - rename: + tag: rename_json_data_primaryAdminChangedEvent_previousAdminEmail_to_google_workspace_alert_data_primary_admin_changed_event_previous_admin_email_239e9fca field: json.data.primaryAdminChangedEvent.previousAdminEmail target_field: google_workspace.alert.data.primary.admin.changed_event.previous_admin_email ignore_missing: true - rename: + tag: rename_json_data_primaryAdminChangedEvent_updatedAdminEmail_to_google_workspace_alert_data_primary_admin_changed_event_updated_admin_email_5e58918a field: json.data.primaryAdminChangedEvent.updatedAdminEmail target_field: google_workspace.alert.data.primary.admin.changed_event.updated_admin_email ignore_missing: true - rename: + tag: rename_json_data_ssoProfileCreatedEvent_inboundSsoProfileName_to_google_workspace_alert_data_sso_profile_created_event_inbound_sso_profile_name_33b6b357 field: json.data.ssoProfileCreatedEvent.inboundSsoProfileName target_field: google_workspace.alert.data.sso_profile.created_event.inbound_sso.profile_name ignore_missing: true - rename: + tag: rename_json_data_ssoProfileUpdatedEvent_inboundSsoProfileName_to_google_workspace_alert_data_sso_profile_updated_event_inbound_sso_profile_name_fa540bbf field: json.data.ssoProfileUpdatedEvent.inboundSsoProfileName target_field: google_workspace.alert.data.sso_profile.updated_event.inbound_sso.profile_name ignore_missing: true - rename: + tag: rename_json_data_ssoProfileUpdatedEvent_inboundSsoProfileChanges_to_google_workspace_alert_data_sso_profile_updated_event_inbound_sso_profile_changes_25b5fc89 field: json.data.ssoProfileUpdatedEvent.inboundSsoProfileChanges target_field: google_workspace.alert.data.sso_profile.updated_event.inbound_sso.profile_changes ignore_missing: true - rename: + tag: rename_json_data_ssoProfileDeletedEvent_inboundSsoProfileName_to_google_workspace_alert_data_sso_profile_deleted_event_inbound_sso_profile_name_c7cfca67 field: json.data.ssoProfileDeletedEvent.inboundSsoProfileName target_field: google_workspace.alert.data.sso_profile.deleted_event.inbound_sso.profile_name ignore_missing: true - rename: + tag: rename_json_data_superAdminPasswordResetEvent_userEmail_to_google_workspace_alert_data_super_admin_password_reset_event_user_email_e64981bd field: json.data.superAdminPasswordResetEvent.userEmail target_field: google_workspace.alert.data.super_admin_password_reset_event.user.email ignore_missing: true - append: + tag: append_related_ip_25216123 field: related.ip value: - '{{{google_workspace.alert.data.login_details.ip_address}}}' @@ -983,6 +1149,7 @@ processors: allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_b7d457ca field: related.user value: - '{{{user.name}}}' @@ -990,6 +1157,7 @@ processors: allow_duplicates: false ignore_failure: true - foreach: + tag: foreach_email_attachments_file_hash_sha256_086756f1 field: email.attachments.file.hash.sha256 processor: append: @@ -1000,6 +1168,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_google_workspace_alert_data_messages_md5_hash_message_body_a87ba08d field: google_workspace.alert.data.messages.md5.hash.message_body processor: append: @@ -1010,10 +1179,12 @@ processors: ignore_missing: true ignore_failure: true - remove: + tag: remove_e5a80934 field: - json ignore_missing: true - remove: + tag: remove_f793838c field: - google_workspace.alert.create_time - google_workspace.alert.customer.id @@ -1036,6 +1207,7 @@ processors: ignore_failure: true ignore_missing: true - foreach: + tag: foreach_google_workspace_alert_data_messages_62c84f33 field: google_workspace.alert.data.messages processor: remove: @@ -1051,6 +1223,7 @@ processors: ignore_missing: true ignore_failure: true - script: + tag: script_e2ccbdd7 description: Drops null/empty values recursively. lang: painless source: | diff --git a/packages/google_workspace/data_stream/calendar/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/calendar/elasticsearch/ingest_pipeline/default.yml index f0497f562a3..56b4ed11ed2 100644 --- a/packages/google_workspace/data_stream/calendar/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/calendar/elasticsearch/ingest_pipeline/default.yml @@ -48,6 +48,7 @@ processors: if: ctx.event?.original != null on_failure: - append: + tag: append_error_message_04014d20 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -79,9 +80,11 @@ processors: } on_failure: - append: + tag: append_error_message_509a956c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: + tag: remove_json_events_parameters_2eab3501 field: json.events.parameters ignore_missing: true - rename: @@ -139,6 +142,7 @@ processors: } on_failure: - append: + tag: append_error_message_6fc3c6d2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -195,6 +199,7 @@ processors: } on_failure: - append: + tag: append_error_message_9475f832 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -202,7 +207,8 @@ processors: tag: rename_json_kind target_field: google_workspace.kind ignore_missing: true - - rename: + - rename: + tag: rename_json_ownerDomain_to_google_workspace_organization_domain_6580cd7d field: json.ownerDomain target_field: google_workspace.organization.domain ignore_missing: true @@ -238,6 +244,7 @@ processors: pattern: '%{}@%{source.user.domain}' on_failure: - append: + tag: append_error_message_4450a0e8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -290,6 +297,7 @@ processors: if: ctx.json?.id?.time != null && ctx.json.id.time != '' on_failure: - append: + tag: append_error_message_de0bed15 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -316,6 +324,7 @@ processors: if: ctx.json?.ipAddress != '' on_failure: - append: + tag: append_error_message_4cb1517b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -338,10 +347,12 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true @@ -364,6 +375,7 @@ processors: if: ctx.event?.action != '' on_failure: - append: + tag: append_error_message_32466e0d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -405,6 +417,7 @@ processors: if: ctx.google_workspace?.calendar?.end_time != null && ctx.google_workspace.calendar.end_time != '' on_failure: - append: + tag: append_error_message_9c7e975e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -435,6 +448,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_82962cb6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -460,7 +474,7 @@ processors: - script: description: Convert Gregorian time in seconds to a Unix epoch timestamp in milliseconds. if: ctx.google_workspace?.calendar?.start_time != null - tag: convert_gregorian_time_to_unix_timestamp + tag: script_7ee314f1 lang: painless source: |- long gregorianOffset = 62135683200L; @@ -476,6 +490,7 @@ processors: if: ctx.google_workspace?.calendar?.start_time != null && ctx.google_workspace.calendar.start_time != '' on_failure: - append: + tag: append_error_message_6cc4756f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -485,8 +500,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_google_workspace_calendar_secs_in_advance_bc89fb57 field: google_workspace.calendar.secs_in_advance - append: + tag: append_error_message_3e3f6fc7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -520,6 +537,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ec007b36 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -562,8 +580,10 @@ processors: if: ctx.google_workspace?.calendar?.requested_period_end != null && ctx.google_workspace.calendar.requested_period_end != '' on_failure: - remove: + tag: remove_google_workspace_calendar_requested_period_end_1c3e0cc1 field: google_workspace.calendar.requested_period_end - append: + tag: append_error_message_d0831cda field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -576,8 +596,10 @@ processors: if: ctx.google_workspace?.calendar?.requested_period_start != null && ctx.google_workspace.calendar.requested_period_start != '' on_failure: - remove: + tag: remove_google_workspace_calendar_requested_period_start_84a0628c field: google_workspace.calendar.requested_period_start - append: + tag: append_error_message_652bb6c2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -591,6 +613,7 @@ processors: target_field: google_workspace.calendar.title ignore_missing: true - user_agent: + tag: user_agent_google_workspace_calendar_user_agent_d60ad417 field: google_workspace.calendar.user_agent ignore_missing: true - append: @@ -680,6 +703,7 @@ processors: drop(ctx); on_failure: - append: + tag: append_error_message_0886867f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -688,6 +712,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/google_workspace/data_stream/chat/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/chat/elasticsearch/ingest_pipeline/default.yml index 50b012b0fdb..ee834baff0e 100644 --- a/packages/google_workspace/data_stream/chat/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/chat/elasticsearch/ingest_pipeline/default.yml @@ -48,9 +48,11 @@ processors: if: ctx.event?.original != null on_failure: - append: + tag: append_error_message_04014d20 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - fingerprint: + tag: fingerprint_52623c07 description: Hashes the ID object and uses it as the document id to avoid duplicate events. fields: - json.id @@ -86,9 +88,11 @@ processors: } on_failure: - append: + tag: append_error_message_f4dbaccc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: + tag: remove_json_events_parameters_2eab3501 field: json.events.parameters ignore_missing: true - rename: @@ -135,6 +139,7 @@ processors: } on_failure: - append: + tag: append_error_message_1faeb051 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -185,6 +190,7 @@ processors: } on_failure: - append: + tag: append_error_message_402e5647 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -192,7 +198,8 @@ processors: tag: rename_json_kind target_field: google_workspace.kind ignore_missing: true - - rename: + - rename: + tag: rename_json_ownerDomain_to_google_workspace_organization_domain_6580cd7d field: json.ownerDomain target_field: google_workspace.organization.domain ignore_missing: true @@ -233,6 +240,7 @@ processors: pattern: '%{}@%{source.user.domain}' on_failure: - append: + tag: append_error_message_4450a0e8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -285,6 +293,7 @@ processors: if: ctx.json?.id?.time != null && ctx.json.id.time != '' on_failure: - append: + tag: append_error_message_de0bed15 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -311,6 +320,7 @@ processors: if: ctx.json?.ipAddress != '' on_failure: - append: + tag: append_error_message_4cb1517b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -333,10 +343,12 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true @@ -359,6 +371,7 @@ processors: if: ctx.event?.action != '' on_failure: - append: + tag: append_error_message_32466e0d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - uri_parts: @@ -367,6 +380,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c593310f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -400,6 +414,7 @@ processors: allow_duplicates: false if: ctx.google_workspace?.chat?.actor != null - foreach: + tag: foreach_google_workspace_chat_target_users_6c7aab15 field: google_workspace.chat.target_users if: ctx.google_workspace?.chat?.target_users instanceof List processor: @@ -472,6 +487,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/google_workspace/data_stream/chrome/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/chrome/elasticsearch/ingest_pipeline/default.yml index b3df59998f9..a8213bec23d 100644 --- a/packages/google_workspace/data_stream/chrome/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/chrome/elasticsearch/ingest_pipeline/default.yml @@ -48,6 +48,7 @@ processors: if: ctx.event?.original != null on_failure: - append: + tag: append_error_message_04014d20 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -79,6 +80,7 @@ processors: } on_failure: - append: + tag: append_error_message_bb1a78dc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -114,6 +116,7 @@ processors: ctx.event.outcome = params.get(ctx.google_workspace.chrome.event_result.toLowerCase()); on_failure: - append: + tag: append_error_message_0df94f18 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -122,6 +125,7 @@ processors: copy_from: google_workspace.chrome.url ignore_empty_value: true - uri_parts: + tag: uri_parts_url_full_62b0b3a9 field: url.full ignore_failure: true - script: @@ -156,6 +160,7 @@ processors: } on_failure: - append: + tag: append_error_message_5b17d1f6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -191,6 +196,7 @@ processors: } on_failure: - append: + tag: append_error_message_a0d69b22 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -220,8 +226,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_google_workspace_chrome_is_encrypted_f9bcb9b2 field: google_workspace.chrome.is_encrypted - append: + tag: append_error_message_e6d9bddc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - split: @@ -232,6 +240,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_0591a88f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - trim: @@ -239,6 +248,7 @@ processors: tag: trim_local_ip ignore_missing: true - foreach: + tag: foreach_google_workspace_chrome_local_ip_329b4ed8 field: google_workspace.chrome.local_ip if: ctx.google_workspace?.chrome?.local_ip instanceof List processor: @@ -262,6 +272,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c19aaaa6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - trim: @@ -269,6 +280,7 @@ processors: tag: trim_remote_ip ignore_missing: true - foreach: + tag: foreach_google_workspace_chrome_remote_ip_0c6f0b5f field: google_workspace.chrome.remote_ip if: ctx.google_workspace?.chrome?.remote_ip instanceof List processor: @@ -293,6 +305,7 @@ processors: if: ctx.json?.ipAddress != '' on_failure: - append: + tag: append_error_message_6938c880 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -305,6 +318,7 @@ processors: if: ctx.json?.id?.time != null && ctx.json.id.time != '' on_failure: - append: + tag: append_error_message_6b9eb14c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -350,8 +364,10 @@ processors: if: ctx.google_workspace?.chrome?.is_federated != '' on_failure: - remove: + tag: remove_google_workspace_chrome_is_federated_e0eaa08c field: google_workspace.chrome.is_federated - append: + tag: append_error_message_73e5a522 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -362,8 +378,10 @@ processors: if: ctx.google_workspace?.chrome?.content_size != '' on_failure: - remove: + tag: remove_google_workspace_chrome_content_size_16f38772 field: google_workspace.chrome.content_size - append: + tag: append_error_message_2953473c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -376,8 +394,10 @@ processors: if: ctx.google_workspace?.chrome?.timestamp != null && ctx.google_workspace.chrome.timestamp != '' on_failure: - remove: + tag: remove_google_workspace_chrome_timestamp_500aab99 field: google_workspace.chrome.timestamp - append: + tag: append_error_message_45a9f91c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -423,6 +443,7 @@ processors: allow_duplicates: false if: ctx.google_workspace?.chrome?.ip_address != null - foreach: + tag: foreach_google_workspace_chrome_local_ip_8c40385f field: google_workspace.chrome.local_ip if: ctx.google_workspace?.chrome?.local_ip instanceof List processor: @@ -432,6 +453,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - foreach: + tag: foreach_google_workspace_chrome_remote_ip_75f2424c field: google_workspace.chrome.remote_ip if: ctx.google_workspace?.chrome?.remote_ip instanceof List processor: @@ -491,6 +513,7 @@ processors: copy_from: google_workspace.chrome.device_id ignore_empty_value: true - user_agent: + tag: user_agent_google_workspace_chrome_user_agent_5e327613 field: google_workspace.chrome.user_agent ignore_missing: true - set: @@ -556,6 +579,7 @@ processors: if: ctx.event?.action != '' on_failure: - append: + tag: append_error_message_ce1e3af4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - join: @@ -565,6 +589,7 @@ processors: if: ctx.event?.action instanceof List on_failure: - append: + tag: append_error_message_b7327b72 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -592,10 +617,12 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true @@ -665,6 +692,7 @@ processors: drop(ctx); on_failure: - append: + tag: append_error_message_0886867f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: @@ -677,6 +705,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/google_workspace/data_stream/context_aware_access/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/context_aware_access/elasticsearch/ingest_pipeline/default.yml index f2fe01fbad0..82073ab4250 100644 --- a/packages/google_workspace/data_stream/context_aware_access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/context_aware_access/elasticsearch/ingest_pipeline/default.yml @@ -13,23 +13,28 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - set: + tag: set_ecs_version_135371bc field: ecs.version value: '8.16.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_90c6475f field: event.original target_field: json on_failure: - append: + tag: append_error_message_45b808be field: error.message value: '{{{_ingest.on_failure_message}}}' - drop: @@ -37,6 +42,7 @@ processors: description: Discard events that are missing the target during the split operation and are subsequently returned as the root object. tag: drop_empty_events - date: + tag: date_json_id_time_9350ecd5 field: json.id.time if: ctx.json?.id?.time != null && ctx.json.id.time != '' formats: @@ -48,13 +54,16 @@ processors: - yyyy/MM/dd HH:mm:ss z on_failure: - append: + tag: append_error_message_18a82a92 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_time_01ac69ca field: google_workspace.id.time copy_from: '@timestamp' ignore_empty_value: true - fingerprint: + tag: fingerprint_52623c07 description: Hashes the ID object and uses it as the document id to avoid duplicate events. fields: - json.id @@ -62,73 +71,90 @@ processors: target_field: _id ignore_missing: true - rename: + tag: rename_json_events_name_to_event_action_75ca640e field: json.events.name target_field: event.action ignore_missing: true - set: + tag: set_google_workspace_event_name_69e4d7fd field: google_workspace.event.name copy_from: event.action ignore_empty_value: true - rename: + tag: rename_json_id_applicationName_to_event_provider_fffca35d field: json.id.applicationName target_field: event.provider ignore_missing: true - set: + tag: set_google_workspace_id_application_name_8030da4a field: google_workspace.id.application_name copy_from: event.provider ignore_empty_value: true - convert: + tag: convert_json_id_uniqueQualifier_to_event_id_0239362e field: json.id.uniqueQualifier target_field: event.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_ab6a6ac3 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_unique_qualifier_8c91fe06 field: google_workspace.id.unique_qualifier copy_from: event.id ignore_empty_value: true - rename: + tag: rename_json_actor_email_to_source_user_email_4ee5e9ab field: json.actor.email target_field: source.user.email ignore_missing: true - set: + tag: set_google_workspace_actor_email_9cd77425 field: google_workspace.actor.email copy_from: source.user.email ignore_empty_value: true - set: + tag: set_user_email_371886fa field: user.email copy_from: source.user.email ignore_empty_value: true - convert: + tag: convert_json_actor_profileId_to_source_user_id_e10534f1 field: json.actor.profileId target_field: source.user.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_dc10d8d6 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_actor_profile_id_6b7a4346 field: google_workspace.actor.profile.id copy_from: source.user.id ignore_empty_value: true - convert: + tag: convert_json_ipAddress_to_source_ip_6cbf48a8 field: json.ipAddress target_field: source.ip type: ip ignore_missing: true on_failure: - append: + tag: append_error_message_79a0c21d field: error.message value: '{{{_ingest.on_failure_message}}}' - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -137,59 +163,73 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - set: + tag: set_google_workspace_ip_address_b61dbbba field: google_workspace.ip_address copy_from: source.ip ignore_empty_value: true - rename: + tag: rename_json_kind_to_google_workspace_kind_8c6d9f29 field: json.kind target_field: google_workspace.kind ignore_missing: true - rename: + tag: rename_json_etag_to_google_workspace_etag_4bd4a803 field: json.etag target_field: google_workspace.etag ignore_missing: true - convert: + tag: convert_json_id_customerId_to_organization_id_ffd90a33 field: json.id.customerId target_field: organization.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_5aacea86 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_customer_id_46a3415a field: google_workspace.id.customer.id copy_from: organization.id ignore_empty_value: true - rename: + tag: rename_json_actor_callerType_to_google_workspace_actor_type_308eeffc field: json.actor.callerType target_field: google_workspace.actor.type ignore_missing: true - rename: + tag: rename_json_actor_key_to_google_workspace_actor_key_3f392d49 field: json.actor.key target_field: google_workspace.actor.key ignore_missing: true - rename: + tag: rename_json_ownerDomain_to_google_workspace_organization_domain_6580cd7d field: json.ownerDomain target_field: google_workspace.organization.domain ignore_missing: true - rename: + tag: rename_json_events_type_to_google_workspace_event_type_e73f1e18 field: json.events.type target_field: google_workspace.event.type ignore_missing: true - set: + tag: set_user_id_f89889ac field: user.id copy_from: source.user.id ignore_empty_value: true - script: + tag: script_a3a2cdbf lang: painless description: Extract User name and User domain from Email. if: ctx.source?.user?.email != null && ctx.source.user.email.contains("@") @@ -206,36 +246,43 @@ processors: ctx.user.domain = splitmail[1]; ctx.source.user.domain = splitmail[1]; - append: + tag: append_related_hosts_23e97a1f field: related.hosts value: '{{{user.domain}}}' if: ctx.user?.domain != null allow_duplicates: false - append: + tag: append_related_hosts_ea7c8eda field: related.hosts value: '{{{google_workspace.organization.domain}}}' if: ctx.google_workspace?.organization?.domain != null allow_duplicates: false - append: + tag: append_related_ip_8121c591 field: related.ip value: '{{{source.ip}}}' if: ctx.source?.ip != null allow_duplicates: false - append: + tag: append_related_user_bcb5a560 field: related.user value: '{{{source.user.id}}}' if: ctx.source?.user?.id != null allow_duplicates: false - append: + tag: append_related_user_f745a124 field: related.user value: '{{{source.user.name}}}' if: ctx.source?.user?.name != null allow_duplicates: false - append: + tag: append_related_user_8c49e24b field: related.user value: '{{{source.user.email}}}' if: ctx.source?.user?.name != null allow_duplicates: false - script: + tag: script_0fda2261 lang: painless description: Map Event parameters into key value pair. if: ctx.json?.events?.parameters != null && ctx.json.events.parameters instanceof List @@ -307,26 +354,32 @@ processors: ctx.json.events.parameters = parameters; } - rename: + tag: rename_google_workspace_context_aware_access_CAA_ACCESS_LEVEL_APPLIED_to_google_workspace_context_aware_access_access_level_applied_1a626e7b field: google_workspace.context_aware_access.CAA_ACCESS_LEVEL_APPLIED target_field: google_workspace.context_aware_access.access_level.applied ignore_missing: true - rename: + tag: rename_google_workspace_context_aware_access_CAA_ACCESS_LEVEL_SATISFIED_to_google_workspace_context_aware_access_access_level_satisfied_41d49bb3 field: google_workspace.context_aware_access.CAA_ACCESS_LEVEL_SATISFIED target_field: google_workspace.context_aware_access.access_level.satisfied ignore_missing: true - rename: + tag: rename_google_workspace_context_aware_access_CAA_ACCESS_LEVEL_UNSATISFIED_to_google_workspace_context_aware_access_access_level_unsatisfied_6fbbe753 field: google_workspace.context_aware_access.CAA_ACCESS_LEVEL_UNSATISFIED target_field: google_workspace.context_aware_access.access_level.unsatisfied ignore_missing: true - rename: + tag: rename_google_workspace_context_aware_access_CAA_APPLICATION_to_google_workspace_context_aware_access_application_fc41b86a field: google_workspace.context_aware_access.CAA_APPLICATION target_field: google_workspace.context_aware_access.application ignore_missing: true - rename: + tag: rename_google_workspace_context_aware_access_CAA_DEVICE_ID_to_google_workspace_context_aware_access_device_id_01d5a337 field: google_workspace.context_aware_access.CAA_DEVICE_ID target_field: google_workspace.context_aware_access.device.id ignore_missing: true - rename: + tag: rename_google_workspace_context_aware_access_CAA_DEVICE_STATE_to_google_workspace_context_aware_access_device_state_a1afa0a7 field: google_workspace.context_aware_access.CAA_DEVICE_STATE target_field: google_workspace.context_aware_access.device.state ignore_missing: true @@ -355,10 +408,12 @@ processors: value: info if: ctx.event?.type == null - remove: + tag: remove_e5a80934 field: - json ignore_missing: true - remove: + tag: remove_2a1ed52f if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - google_workspace.ip_address @@ -371,6 +426,7 @@ processors: - google_workspace.id.time ignore_missing: true - script: + tag: script_e2ccbdd7 description: Drops null/empty values recursively. lang: painless source: | @@ -388,10 +444,12 @@ processors: } dropEmptyFields(ctx); - set: + tag: set_event_kind_92954dfa field: event.kind value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/google_workspace/data_stream/data_studio/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/data_studio/elasticsearch/ingest_pipeline/default.yml index 7ca13424aaf..67c4b2eaed8 100644 --- a/packages/google_workspace/data_stream/data_studio/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/data_studio/elasticsearch/ingest_pipeline/default.yml @@ -48,6 +48,7 @@ processors: if: ctx.event?.original != null on_failure: - append: + tag: append_error_message_04014d20 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -79,9 +80,11 @@ processors: } on_failure: - append: + tag: append_error_message_a6a18bcc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: + tag: remove_json_events_parameters_2eab3501 field: json.events.parameters ignore_missing: true - rename: @@ -130,6 +133,7 @@ processors: ctx.event.put('category', category); on_failure: - append: + tag: append_error_message_17895c18 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -165,6 +169,7 @@ processors: ctx.event.put('type', type); on_failure: - append: + tag: append_error_message_45d39e40 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -172,7 +177,8 @@ processors: tag: rename_json_kind target_field: google_workspace.kind ignore_missing: true - - rename: + - rename: + tag: rename_json_ownerDomain_to_google_workspace_organization_domain_6580cd7d field: json.ownerDomain target_field: google_workspace.organization.domain ignore_missing: true @@ -203,6 +209,7 @@ processors: pattern: '%{source.user.name}@%{source.user.domain}' on_failure: - append: + tag: append_error_message_b1c3b26d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -255,6 +262,7 @@ processors: if: ctx.json?.id?.time != null && ctx.json.id.time != '' on_failure: - append: + tag: append_error_message_de0bed15 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -281,6 +289,7 @@ processors: if: ctx.json?.ipAddress != '' on_failure: - append: + tag: append_error_message_4cb1517b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -303,10 +312,12 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true @@ -332,6 +343,7 @@ processors: pattern: '%{user.target.name}@%{user.target.domain}' on_failure: - append: + tag: append_error_message_3d41b0fb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -351,6 +363,7 @@ processors: if: ctx.event?.action != '' on_failure: - append: + tag: append_error_message_cfa0331f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - join: @@ -360,6 +373,7 @@ processors: if: ctx.event?.action instanceof List on_failure: - append: + tag: append_error_message_b7327b72 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -430,6 +444,7 @@ processors: drop(ctx); on_failure: - append: + tag: append_error_message_0886867f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -438,6 +453,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/google_workspace/data_stream/device/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/device/elasticsearch/ingest_pipeline/default.yml index 532e93d7b09..dfdcadeebc4 100644 --- a/packages/google_workspace/data_stream/device/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/device/elasticsearch/ingest_pipeline/default.yml @@ -13,23 +13,28 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - set: + tag: set_ecs_version_135371bc field: ecs.version value: '8.16.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_90c6475f field: event.original target_field: json on_failure: - append: + tag: append_error_message_45b808be field: error.message value: '{{{_ingest.on_failure_message}}}' - drop: @@ -37,6 +42,7 @@ processors: description: Discard events that are missing the target during the split operation and are subsequently returned as the root object. tag: drop_empty_events - date: + tag: date_json_id_time_9350ecd5 field: json.id.time if: ctx.json?.id?.time != null && ctx.json.id.time != '' formats: @@ -48,13 +54,16 @@ processors: - yyyy/MM/dd HH:mm:ss z on_failure: - append: + tag: append_error_message_18a82a92 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_time_01ac69ca field: google_workspace.id.time copy_from: '@timestamp' ignore_empty_value: true - fingerprint: + tag: fingerprint_52623c07 description: Hashes the ID object and uses it as the document id to avoid duplicate events. fields: - json.id @@ -62,74 +71,91 @@ processors: target_field: _id ignore_missing: true - rename: + tag: rename_json_events_name_to_event_action_75ca640e field: json.events.name target_field: event.action ignore_missing: true - set: + tag: set_google_workspace_event_name_69e4d7fd field: google_workspace.event.name copy_from: event.action ignore_empty_value: true - rename: + tag: rename_json_id_applicationName_to_event_provider_fffca35d field: json.id.applicationName target_field: event.provider ignore_missing: true - set: + tag: set_google_workspace_id_application_name_8030da4a field: google_workspace.id.application_name copy_from: event.provider ignore_empty_value: true - convert: + tag: convert_json_id_uniqueQualifier_to_event_id_0239362e field: json.id.uniqueQualifier target_field: event.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_ab6a6ac3 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_unique_qualifier_8c91fe06 field: google_workspace.id.unique_qualifier copy_from: event.id ignore_empty_value: true - rename: + tag: rename_json_actor_email_to_source_user_email_4ee5e9ab field: json.actor.email target_field: source.user.email ignore_missing: true - set: + tag: set_google_workspace_actor_email_9cd77425 field: google_workspace.actor.email copy_from: source.user.email ignore_empty_value: true - append: + tag: append_user_email_a27eecdf field: user.email value: '{{{source.user.email}}}' if: ctx.source?.user?.email != null allow_duplicates: false - convert: + tag: convert_json_actor_profileId_to_source_user_id_e10534f1 field: json.actor.profileId target_field: source.user.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_dc10d8d6 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_actor_profile_id_6b7a4346 field: google_workspace.actor.profile.id copy_from: source.user.id ignore_empty_value: true - convert: + tag: convert_json_ipAddress_to_source_ip_6cbf48a8 field: json.ipAddress target_field: source.ip type: ip ignore_missing: true on_failure: - append: + tag: append_error_message_79a0c21d field: error.message value: '{{{_ingest.on_failure_message}}}' - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -138,59 +164,73 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - set: + tag: set_google_workspace_ip_address_b61dbbba field: google_workspace.ip_address copy_from: source.ip ignore_empty_value: true - rename: + tag: rename_json_kind_to_google_workspace_kind_8c6d9f29 field: json.kind target_field: google_workspace.kind ignore_missing: true - rename: + tag: rename_json_etag_to_google_workspace_etag_4bd4a803 field: json.etag target_field: google_workspace.etag ignore_missing: true - convert: + tag: convert_json_id_customerId_to_organization_id_ffd90a33 field: json.id.customerId target_field: organization.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_5aacea86 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_customer_id_46a3415a field: google_workspace.id.customer.id copy_from: organization.id ignore_empty_value: true - rename: + tag: rename_json_actor_callerType_to_google_workspace_actor_type_308eeffc field: json.actor.callerType target_field: google_workspace.actor.type ignore_missing: true - rename: + tag: rename_json_actor_key_to_google_workspace_actor_key_3f392d49 field: json.actor.key target_field: google_workspace.actor.key ignore_missing: true - rename: + tag: rename_json_ownerDomain_to_google_workspace_organization_domain_6580cd7d field: json.ownerDomain target_field: google_workspace.organization.domain ignore_missing: true - rename: + tag: rename_json_events_type_to_google_workspace_event_type_e73f1e18 field: json.events.type target_field: google_workspace.event.type ignore_missing: true - set: + tag: set_user_id_f89889ac field: user.id copy_from: source.user.id ignore_empty_value: true - script: + tag: script_a3a2cdbf lang: painless description: Extract User name and User domain from Email. if: ctx.source?.user?.email != null && ctx.source.user.email.contains("@") @@ -207,31 +247,37 @@ processors: ctx.user.domain = splitmail[1]; ctx.source.user.domain = splitmail[1]; - append: + tag: append_related_hosts_23e97a1f field: related.hosts value: '{{{user.domain}}}' if: ctx.user?.domain != null allow_duplicates: false - append: + tag: append_related_hosts_ea7c8eda field: related.hosts value: '{{{google_workspace.organization.domain}}}' if: ctx.google_workspace?.organization?.domain != null allow_duplicates: false - append: + tag: append_related_ip_8121c591 field: related.ip value: '{{{source.ip}}}' if: ctx.source?.ip != null allow_duplicates: false - append: + tag: append_related_user_bcb5a560 field: related.user value: '{{{source.user.id}}}' if: ctx.source?.user?.id != null allow_duplicates: false - append: + tag: append_related_user_f745a124 field: related.user value: '{{{source.user.name}}}' if: ctx.source?.user?.name != null allow_duplicates: false - script: + tag: script_21e31442 lang: painless description: Map Event parameters into key value pair. if: ctx.json?.events?.parameters != null && ctx.json.events.parameters instanceof List @@ -303,43 +349,53 @@ processors: ctx.json.events.parameters = parameters; } - rename: + tag: rename_google_workspace_device_ACCOUNT_STATE_to_google_workspace_device_account_state_a5953012 field: google_workspace.device.ACCOUNT_STATE target_field: google_workspace.device.account_state ignore_missing: true - rename: + tag: rename_google_workspace_device_ACTION_EXECUTION_STATUS_to_google_workspace_device_action_execution_status_7490f8d1 field: google_workspace.device.ACTION_EXECUTION_STATUS target_field: google_workspace.device.action.execution_status ignore_missing: true - rename: + tag: rename_google_workspace_device_ACTION_ID_to_google_workspace_device_action_id_79fb3179 field: google_workspace.device.ACTION_ID target_field: google_workspace.device.action.id ignore_missing: true - rename: + tag: rename_google_workspace_device_ACTION_TYPE_to_google_workspace_device_action_type_de6beb8b field: google_workspace.device.ACTION_TYPE target_field: google_workspace.device.action.type ignore_missing: true - rename: + tag: rename_google_workspace_device_APK_SHA256_HASH_to_google_workspace_device_apk_sha256_hash_460a37ea field: google_workspace.device.APK_SHA256_HASH target_field: google_workspace.device.apk_sha256_hash ignore_missing: true - append: + tag: append_related_hash_bf24e73b field: related.hash value: '{{{google_workspace.device.apk_sha256_hash}}}' if: ctx.google_workspace?.device?.apk_sha256_hash != null allow_duplicates: false - rename: + tag: rename_google_workspace_device_APPLICATION_ID_to_google_workspace_device_application_id_43285ce9 field: google_workspace.device.APPLICATION_ID target_field: google_workspace.device.application.id ignore_missing: true - rename: + tag: rename_google_workspace_device_APPLICATION_MESSAGE_to_google_workspace_device_application_message_cf9909d7 field: google_workspace.device.APPLICATION_MESSAGE target_field: google_workspace.device.application.message ignore_missing: true - rename: + tag: rename_google_workspace_device_APPLICATION_REPORT_KEY_to_google_workspace_device_application_report_key_cc27c882 field: google_workspace.device.APPLICATION_REPORT_KEY target_field: google_workspace.device.application.report.key ignore_missing: true - date: + tag: date_google_workspace_device_APPLICATION_REPORT_TIMESTAMP_to_google_workspace_device_application_report_timestamp_299ff761 field: google_workspace.device.APPLICATION_REPORT_TIMESTAMP target_field: google_workspace.device.application.report.timestamp if: ctx.google_workspace?.device?.APPLICATION_REPORT_TIMESTAMP != null && ctx.google_workspace.device.APPLICATION_REPORT_TIMESTAMP != '' @@ -352,6 +408,7 @@ processors: - yyyy/MM/dd HH:mm:ss z on_failure: - append: + tag: append_error_message_eb65090e field: error.message value: '{{{_ingest.on_failure_message}}}' - date: @@ -369,164 +426,204 @@ processors: if: ctx.google_workspace?.device?.LAST_SYNC_AUDIT_DATE != null && ctx.google_workspace.device.LAST_SYNC_AUDIT_DATE != '' on_failure: - append: + tag: append_error_message_139625b1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_google_workspace_device_APPLICATION_REPORT_SEVERITY_to_google_workspace_device_application_report_severity_299fe304 field: google_workspace.device.APPLICATION_REPORT_SEVERITY target_field: google_workspace.device.application.report.severity ignore_missing: true - rename: + tag: rename_google_workspace_device_APPLICATION_STATE_to_google_workspace_device_application_state_0ae09ac3 field: google_workspace.device.APPLICATION_STATE target_field: google_workspace.device.application.state ignore_missing: true - rename: + tag: rename_google_workspace_device_BASIC_INTEGRITY_to_google_workspace_device_basic_integrity_39c9b8e4 field: google_workspace.device.BASIC_INTEGRITY target_field: google_workspace.device.basic_integrity ignore_missing: true - rename: + tag: rename_google_workspace_device_CTS_PROFILE_MATCH_to_google_workspace_device_cts_profile_match_d61d4c00 field: google_workspace.device.CTS_PROFILE_MATCH target_field: google_workspace.device.cts_profile_match ignore_missing: true - rename: + tag: rename_google_workspace_device_DEVICE_COMPLIANCE_to_google_workspace_device_compliance_55661d89 field: google_workspace.device.DEVICE_COMPLIANCE target_field: google_workspace.device.compliance ignore_missing: true - rename: + tag: rename_google_workspace_device_DEVICE_COMPROMISED_STATE_to_google_workspace_device_compromised_state_fe2bcc6d field: google_workspace.device.DEVICE_COMPROMISED_STATE target_field: google_workspace.device.compromised_state ignore_missing: true - rename: + tag: rename_google_workspace_device_DEVICE_DEACTIVATION_REASON_to_google_workspace_device_deactivation_reason_ab15eee5 field: google_workspace.device.DEVICE_DEACTIVATION_REASON target_field: google_workspace.device.deactivation_reason ignore_missing: true - rename: + tag: rename_google_workspace_device_DEVICE_ID_to_google_workspace_device_id_6f0bacbd field: google_workspace.device.DEVICE_ID target_field: google_workspace.device.id ignore_missing: true - rename: + tag: rename_google_workspace_device_DEVICE_MODEL_to_google_workspace_device_model_c09b9f6b field: google_workspace.device.DEVICE_MODEL target_field: google_workspace.device.model ignore_missing: true - rename: + tag: rename_google_workspace_device_DEVICE_OWNERSHIP_to_google_workspace_device_ownership_9f193e1f field: google_workspace.device.DEVICE_OWNERSHIP target_field: google_workspace.device.ownership ignore_missing: true - rename: + tag: rename_google_workspace_device_DEVICE_PROPERTY_to_google_workspace_device_property_6795c31d field: google_workspace.device.DEVICE_PROPERTY target_field: google_workspace.device.property ignore_missing: true - rename: + tag: rename_google_workspace_device_DEVICE_SETTING_to_google_workspace_device_setting_caf78ef5 field: google_workspace.device.DEVICE_SETTING target_field: google_workspace.device.setting ignore_missing: true - rename: + tag: rename_google_workspace_device_DEVICE_STATUS_ON_APPLE_PORTAL_to_google_workspace_device_status_on_apple_portal_96bfe4b1 field: google_workspace.device.DEVICE_STATUS_ON_APPLE_PORTAL target_field: google_workspace.device.status_on_apple_portal ignore_missing: true - rename: + tag: rename_google_workspace_device_DEVICE_TYPE_to_google_workspace_device_type_4fca0cfd field: google_workspace.device.DEVICE_TYPE target_field: google_workspace.device.type ignore_missing: true - convert: + tag: convert_google_workspace_device_FAILED_PASSWD_ATTEMPTS_to_google_workspace_device_failed_passwd_attempts_53693b20 field: google_workspace.device.FAILED_PASSWD_ATTEMPTS target_field: google_workspace.device.failed_passwd_attempts type: long ignore_missing: true on_failure: - append: + tag: append_error_message_369403e3 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_google_workspace_device_IOS_VENDOR_ID_to_google_workspace_device_ios_vendor_id_7adfce78 field: google_workspace.device.IOS_VENDOR_ID target_field: google_workspace.device.ios_vendor_id ignore_missing: true - rename: + tag: rename_google_workspace_device_NEW_DEVICE_ID_to_google_workspace_device_new_device_id_c04e20ce field: google_workspace.device.NEW_DEVICE_ID target_field: google_workspace.device.new_device_id ignore_missing: true - rename: + tag: rename_google_workspace_device_NEW_VALUE_to_google_workspace_device_new_value_b285f560 field: google_workspace.device.NEW_VALUE target_field: google_workspace.device.new_value ignore_missing: true - rename: + tag: rename_google_workspace_device_OLD_VALUE_to_google_workspace_device_old_value_4fbafaaa field: google_workspace.device.OLD_VALUE target_field: google_workspace.device.old_value ignore_missing: true - rename: + tag: rename_google_workspace_device_OS_EDITION_to_google_workspace_device_os_edition_3d364d43 field: google_workspace.device.OS_EDITION target_field: google_workspace.device.os.edition ignore_missing: true - rename: + tag: rename_google_workspace_device_OS_PROPERTY_to_google_workspace_device_os_property_b7fb8ad1 field: google_workspace.device.OS_PROPERTY target_field: google_workspace.device.os.property ignore_missing: true - rename: + tag: rename_google_workspace_device_OS_VERSION_to_google_workspace_device_os_version_8f27163b field: google_workspace.device.OS_VERSION target_field: google_workspace.device.os.version ignore_missing: true - set: + tag: set_host_os_version_4df4bd2c field: host.os.version copy_from: google_workspace.device.os.version ignore_empty_value: true - rename: + tag: rename_google_workspace_device_PHA_CATEGORY_to_google_workspace_device_pha_category_316fd108 field: google_workspace.device.PHA_CATEGORY target_field: google_workspace.device.pha_category ignore_missing: true - rename: + tag: rename_google_workspace_device_POLICY_NAME_to_google_workspace_device_policy_name_10804891 field: google_workspace.device.POLICY_NAME target_field: google_workspace.device.policy.name ignore_missing: true - rename: + tag: rename_google_workspace_device_POLICY_SYNC_RESULT_to_google_workspace_device_policy_sync_result_ae1504c6 field: google_workspace.device.POLICY_SYNC_RESULT target_field: google_workspace.device.policy.sync.result ignore_missing: true - rename: + tag: rename_google_workspace_device_POLICY_SYNC_TYPE_to_google_workspace_device_policy_sync_type_d13c572e field: google_workspace.device.POLICY_SYNC_TYPE target_field: google_workspace.device.policy.sync.type ignore_missing: true - rename: + tag: rename_google_workspace_device_REGISTER_PRIVILEGE_to_google_workspace_device_register_privilege_a7b986b0 field: google_workspace.device.REGISTER_PRIVILEGE target_field: google_workspace.device.register_privilege ignore_missing: true - rename: + tag: rename_google_workspace_device_RESOURCE_ID_to_google_workspace_device_resource_id_40b3e86d field: google_workspace.device.RESOURCE_ID target_field: google_workspace.device.resource.id ignore_missing: true - rename: + tag: rename_google_workspace_device_RISK_SIGNAL_to_google_workspace_device_risk_signal_b22ed3ec field: google_workspace.device.RISK_SIGNAL target_field: google_workspace.device.risk_signal ignore_missing: true - convert: + tag: convert_google_workspace_device_SECURITY_EVENT_ID_to_google_workspace_device_security_event_id_44d581a1 field: google_workspace.device.SECURITY_EVENT_ID target_field: google_workspace.device.security.event_id type: long ignore_missing: true on_failure: - append: + tag: append_error_message_bd15c3f2 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_google_workspace_device_SECURITY_PATCH_LEVEL_to_google_workspace_device_security_patch_level_d099bf57 field: google_workspace.device.SECURITY_PATCH_LEVEL target_field: google_workspace.device.security.patch_level ignore_missing: true - rename: + tag: rename_google_workspace_device_SERIAL_NUMBER_to_google_workspace_device_serial_number_e40abc28 field: google_workspace.device.SERIAL_NUMBER target_field: google_workspace.device.serial_number ignore_missing: true - rename: + tag: rename_google_workspace_device_USER_EMAIL_to_google_workspace_device_user_email_192f08a8 field: google_workspace.device.USER_EMAIL target_field: google_workspace.device.user_email ignore_missing: true - append: + tag: append_user_email_daee5827 field: user.email value: '{{{google_workspace.device.user_email}}}' if: ctx.google_workspace?.device?.user_email != null allow_duplicates: false - rename: + tag: rename_google_workspace_device_VALUE_to_google_workspace_device_value_23b0d792 field: google_workspace.device.VALUE target_field: google_workspace.device.value ignore_missing: true - rename: + tag: rename_google_workspace_device_WINDOWS_SYNCML_POLICY_STATUS_CODE_to_google_workspace_device_windows_syncml_policy_status_code_9f8183c4 field: google_workspace.device.WINDOWS_SYNCML_POLICY_STATUS_CODE target_field: google_workspace.device.windows_syncml_policy_status_code ignore_missing: true @@ -652,6 +749,7 @@ processors: ctx.google_workspace?.device?.failed_passwd_attempts != null && ctx.google_workspace.device.failed_passwd_attempts > 0 - foreach: + tag: foreach_user_email_88381b88 field: user.email if: ctx.user?.email instanceof List processor: @@ -661,6 +759,7 @@ processors: allow_duplicates: false ignore_missing: true - remove: + tag: remove_38152ac7 field: - json - google_workspace.device.APPLICATION_REPORT_TIMESTAMP @@ -669,6 +768,7 @@ processors: - google_workspace.device.LAST_SYNC_AUDIT_DATE ignore_missing: true - remove: + tag: remove_b2562779 if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - google_workspace.ip_address @@ -683,6 +783,7 @@ processors: - google_workspace.id.time ignore_missing: true - script: + tag: script_e2ccbdd7 description: Drops null/empty values recursively. lang: painless source: | @@ -700,10 +801,12 @@ processors: } dropEmptyFields(ctx); - set: + tag: set_event_kind_92954dfa field: event.kind value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/google_workspace/data_stream/drive/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/drive/elasticsearch/ingest_pipeline/default.yml index 273519a99e8..26d3e63f8cd 100644 --- a/packages/google_workspace/data_stream/drive/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/drive/elasticsearch/ingest_pipeline/default.yml @@ -13,25 +13,31 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - set: + tag: set_ecs_version_135371bc field: ecs.version value: '8.16.0' - append: + tag: append_event_category_423f7188 field: event.category value: file - set: + tag: set_event_kind_de80643c field: event.kind value: event - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - drop: @@ -39,6 +45,7 @@ processors: description: Discard events that are missing the target during the split operation and are subsequently returned as the root object. tag: drop_empty_events - date: + tag: date_json_id_time_dc34ff3c field: json.id.time if: ctx.json?.id?.time != null && ctx.json.id.time != '' timezone: UTC @@ -50,9 +57,11 @@ processors: - yyyy/MM/dd HH:mm:ss z on_failure: - append: + tag: append_error_message_a48f8d5f field: error.message value: "{{{_ingest.on_failure_message}}}" - fingerprint: + tag: fingerprint_567b61c4 description: Hashes the ID object and uses it as the document id to avoid duplicate events. fields: - json.id @@ -61,40 +70,49 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_events_name_to_event_action_75ca640e field: json.events.name target_field: event.action ignore_missing: true - rename: + tag: rename_json_id_applicationName_to_event_provider_fffca35d field: json.id.applicationName target_field: event.provider ignore_missing: true - convert: + tag: convert_json_id_uniqueQualifier_to_event_id_0664525d field: json.id.uniqueQualifier target_field: event.id type: string ignore_missing: true - rename: + tag: rename_json_actor_email_to_source_user_email_4ee5e9ab field: json.actor.email target_field: source.user.email ignore_missing: true - set: + tag: set_user_email_f8429233 field: user.email copy_from: source.user.email ignore_failure: true - convert: + tag: convert_json_actor_profileId_to_source_user_id_edfcffb4 field: json.actor.profileId target_field: source.user.id type: string ignore_missing: true - set: + tag: set_user_id_64f70b73 field: user.id copy_from: source.user.id if: ctx?.source?.user?.id != null - rename: + tag: rename_json_ipAddress_to_source_ip_726bbe73 field: json.ipAddress target_field: source.ip ignore_missing: true - rename: + tag: rename_json_kind_to_google_workspace_kind_8c6d9f29 field: json.kind target_field: google_workspace.kind ignore_missing: true @@ -105,29 +123,36 @@ processors: tag: rename_customer_id_to_organization_id on_failure: - append: + tag: append_error_message_0678effb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_actor_callerType_to_google_workspace_actor_type_308eeffc field: json.actor.callerType target_field: google_workspace.actor.type ignore_missing: true - rename: + tag: rename_json_actor_key_to_google_workspace_actor_key_3f392d49 field: json.actor.key target_field: google_workspace.actor.key ignore_missing: true - rename: + tag: rename_json_actor_applicationInfo_applicationName_to_google_workspace_actor_application_name_7bd54cfa field: json.actor.applicationInfo.applicationName target_field: google_workspace.actor.application_name ignore_missing: true - rename: + tag: rename_json_ownerDomain_to_google_workspace_organization_domain_6580cd7d field: json.ownerDomain target_field: google_workspace.organization.domain ignore_missing: true - rename: + tag: rename_json_events_type_to_google_workspace_event_type_e73f1e18 field: json.events.type target_field: google_workspace.event.type ignore_missing: true - script: + tag: script_8d756e1a lang: painless if: 'ctx?.source?.user?.email != null && ctx?.source?.user?.email.contains("@")' source: > @@ -142,31 +167,38 @@ processors: ctx.source.user.name = splitmail[0]; ctx.user.domain = splitmail[1]; ctx.source.user.domain = splitmail[1]; - - append: + - append: + tag: append_event_type_aadb53b4 field: event.type value: change if: '["add_to_folder", "edit", "add_lock", "move", "remove_from_folder", "rename", "remove_lock", "sheets_import_range", "approval_canceled", "approval_comment_added", "approval_requested", "approval_reviewer_responded", "change_acl_editors", "change_document_access_scope", "change_document_visibility", "shared_drive_membership_change", "shared_drive_settings_change", "sheets_import_range_access_change", "change_user_access"].contains(ctx?.event?.action)' - - append: + - append: + tag: append_event_category_842af8a1 field: event.category value: iam if: '["approval_canceled", "approval_comment_added", "approval_requested", "approval_reviewer_responded", "change_acl_editors", "change_document_access_scope", "change_document_visibility", "shared_drive_membership_change", "shared_drive_settings_change", "sheets_import_range_access_change", "change_user_access"].contains(ctx?.event?.action)' - - append: + - append: + tag: append_event_category_ea3f2734 field: event.category value: configuration if: '["approval_canceled", "approval_comment_added", "approval_requested", "approval_reviewer_responded", "change_acl_editors", "change_document_access_scope", "change_document_visibility", "shared_drive_membership_change", "shared_drive_settings_change", "sheets_import_range_access_change", "change_user_access"].contains(ctx?.event?.action)' - - append: + - append: + tag: append_event_type_64b4a5c3 field: event.type value: creation if: '["create", "untrash", "upload"].contains(ctx?.event?.action)' - - append: + - append: + tag: append_event_type_c187e617 field: event.type value: deletion if: '["delete", "trash"].contains(ctx?.event?.action)' - - append: + - append: + tag: append_event_type_f24896a2 field: event.type value: info if: '["download", "preview", "print", "view"].contains(ctx?.event?.action)' - script: + tag: script_494eab2f lang: painless if: 'ctx?.json?.events?.parameters != null && ctx?.json?.events?.parameters instanceof List' source: > @@ -185,47 +217,58 @@ processors: } } - rename: + tag: rename_google_workspace_drive_doc_id_to_google_workspace_drive_file_id_31e48411 field: google_workspace.drive.doc_id target_field: google_workspace.drive.file.id ignore_missing: true - rename: + tag: rename_google_workspace_drive_doc_title_to_file_name_ce1d1ffb field: google_workspace.drive.doc_title target_field: file.name ignore_missing: true - rename: + tag: rename_google_workspace_drive_doc_type_to_google_workspace_drive_file_type_d86db149 field: google_workspace.drive.doc_type target_field: google_workspace.drive.file.type ignore_missing: true - rename: + tag: rename_google_workspace_drive_owner_to_google_workspace_drive_file_owner_email_a47fe15c field: google_workspace.drive.owner target_field: google_workspace.drive.file.owner.email ignore_missing: true - rename: + tag: rename_google_workspace_drive_owner_is_shared_drive_to_google_workspace_drive_file_owner_is_shared_drive_547c3311 field: google_workspace.drive.owner_is_shared_drive target_field: google_workspace.drive.file.owner.is_shared_drive ignore_missing: true - rename: + tag: rename_google_workspace_drive_new_settings_state_to_google_workspace_drive_new_value_3cdce618 field: google_workspace.drive.new_settings_state target_field: google_workspace.drive.new_value ignore_missing: true - rename: + tag: rename_google_workspace_drive_old_settings_state_to_google_workspace_drive_old_value_a7881d42 field: google_workspace.drive.old_settings_state target_field: google_workspace.drive.old_value ignore_missing: true - rename: + tag: rename_google_workspace_drive_target_user_to_google_workspace_drive_target_33c37ec1 field: google_workspace.drive.target_user target_field: google_workspace.drive.target if: ctx.google_workspace?.drive?.target == null ignore_missing: true - set: + tag: set_file_type_e4bb631b field: file.type value: dir if: '["folder", "shared"].contains(ctx?.google_workspace?.drive?.file?.type)' - set: + tag: set_file_type_3c2c5e73 field: file.type value: file if: ctx?.file?.type == null - script: + tag: script_571619be lang: painless if: ctx?.file?.name != null source: > @@ -235,6 +278,7 @@ processors: ctx.file.extension = path.substring(extIdx+1); } - script: + tag: script_72693fa0 lang: painless if: 'ctx?.google_workspace?.drive?.target != null && ctx?.google_workspace?.drive?.target.contains("@")' source: > @@ -250,6 +294,7 @@ processors: } ctx.related.user.add(splitmail[0]); - script: + tag: script_125fa133 lang: painless if: 'ctx?.google_workspace?.drive?.file?.owner?.email != null && ctx?.google_workspace?.drive?.file?.owner?.email.contains("@")' source: > @@ -262,25 +307,30 @@ processors: } ctx.file.owner = splitmail[0]; - append: + tag: append_related_ip_549fa858 field: related.ip value: "{{{source.ip}}}" if: ctx?.source?.ip != null allow_duplicates: false - append: + tag: append_related_user_bed17bbe field: related.user value: "{{{file.owner}}}" if: ctx?.file?.owner != null allow_duplicates: false - append: + tag: append_related_user_8302ba1f field: related.user value: "{{{source.user.name}}}" if: ctx?.source?.user?.name != null allow_duplicates: false - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -289,15 +339,18 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - remove: - field: + tag: remove_e5a80934 + field: - json ignore_missing: true on_failure: diff --git a/packages/google_workspace/data_stream/gcp/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/gcp/elasticsearch/ingest_pipeline/default.yml index b55c5b55d5d..583b96e9afc 100644 --- a/packages/google_workspace/data_stream/gcp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/gcp/elasticsearch/ingest_pipeline/default.yml @@ -13,23 +13,28 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - set: + tag: set_ecs_version_135371bc field: ecs.version value: '8.16.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_90c6475f field: event.original target_field: json on_failure: - append: + tag: append_error_message_45b808be field: error.message value: '{{{_ingest.on_failure_message}}}' - drop: @@ -41,6 +46,7 @@ processors: tag: set_event_kind value: event - date: + tag: date_json_id_time_9350ecd5 field: json.id.time if: ctx.json?.id?.time != null && ctx.json.id.time != '' formats: @@ -52,13 +58,16 @@ processors: - yyyy/MM/dd HH:mm:ss z on_failure: - append: + tag: append_error_message_18a82a92 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_time_01ac69ca field: google_workspace.id.time copy_from: '@timestamp' ignore_empty_value: true - fingerprint: + tag: fingerprint_52623c07 description: Hashes the ID object and uses it as the document id to avoid duplicate events. fields: - json.id @@ -66,73 +75,90 @@ processors: target_field: _id ignore_missing: true - rename: + tag: rename_json_events_name_to_event_action_75ca640e field: json.events.name target_field: event.action ignore_missing: true - set: + tag: set_google_workspace_event_name_69e4d7fd field: google_workspace.event.name copy_from: event.action ignore_empty_value: true - rename: + tag: rename_json_id_applicationName_to_event_provider_fffca35d field: json.id.applicationName target_field: event.provider ignore_missing: true - set: + tag: set_google_workspace_id_application_name_8030da4a field: google_workspace.id.application_name copy_from: event.provider ignore_empty_value: true - convert: + tag: convert_json_id_uniqueQualifier_to_event_id_0239362e field: json.id.uniqueQualifier target_field: event.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_ab6a6ac3 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_unique_qualifier_8c91fe06 field: google_workspace.id.unique_qualifier copy_from: event.id ignore_empty_value: true - rename: + tag: rename_json_actor_email_to_source_user_email_4ee5e9ab field: json.actor.email target_field: source.user.email ignore_missing: true - set: + tag: set_google_workspace_actor_email_9cd77425 field: google_workspace.actor.email copy_from: source.user.email ignore_empty_value: true - set: + tag: set_user_email_371886fa field: user.email copy_from: source.user.email ignore_empty_value: true - convert: + tag: convert_json_actor_profileId_to_source_user_id_e10534f1 field: json.actor.profileId target_field: source.user.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_dc10d8d6 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_actor_profile_id_6b7a4346 field: google_workspace.actor.profile.id copy_from: source.user.id ignore_empty_value: true - convert: + tag: convert_json_ipAddress_to_source_ip_6cbf48a8 field: json.ipAddress target_field: source.ip type: ip ignore_missing: true on_failure: - append: + tag: append_error_message_79a0c21d field: error.message value: '{{{_ingest.on_failure_message}}}' - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -141,59 +167,73 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - set: + tag: set_google_workspace_ip_address_b61dbbba field: google_workspace.ip_address copy_from: source.ip ignore_empty_value: true - rename: + tag: rename_json_kind_to_google_workspace_kind_8c6d9f29 field: json.kind target_field: google_workspace.kind ignore_missing: true - rename: + tag: rename_json_etag_to_google_workspace_etag_4bd4a803 field: json.etag target_field: google_workspace.etag ignore_missing: true - convert: + tag: convert_json_id_customerId_to_organization_id_ffd90a33 field: json.id.customerId target_field: organization.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_5aacea86 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_customer_id_46a3415a field: google_workspace.id.customer.id copy_from: organization.id ignore_empty_value: true - rename: + tag: rename_json_actor_callerType_to_google_workspace_actor_type_308eeffc field: json.actor.callerType target_field: google_workspace.actor.type ignore_missing: true - rename: + tag: rename_json_actor_key_to_google_workspace_actor_key_3f392d49 field: json.actor.key target_field: google_workspace.actor.key ignore_missing: true - rename: + tag: rename_json_ownerDomain_to_google_workspace_organization_domain_6580cd7d field: json.ownerDomain target_field: google_workspace.organization.domain ignore_missing: true - rename: + tag: rename_json_events_type_to_google_workspace_event_type_e73f1e18 field: json.events.type target_field: google_workspace.event.type ignore_missing: true - set: + tag: set_user_id_f89889ac field: user.id copy_from: source.user.id ignore_empty_value: true - script: + tag: script_a3a2cdbf lang: painless description: Extract User name and User domain from Email. if: ctx.source?.user?.email != null && ctx.source.user.email.contains("@") @@ -210,31 +250,37 @@ processors: ctx.user.domain = splitmail[1]; ctx.source.user.domain = splitmail[1]; - append: + tag: append_related_hosts_23e97a1f field: related.hosts value: '{{{user.domain}}}' if: ctx.user?.domain != null allow_duplicates: false - append: + tag: append_related_hosts_ea7c8eda field: related.hosts value: '{{{google_workspace.organization.domain}}}' if: ctx.google_workspace?.organization?.domain != null allow_duplicates: false - append: + tag: append_related_ip_8121c591 field: related.ip value: '{{{source.ip}}}' if: ctx.source?.ip != null allow_duplicates: false - append: + tag: append_related_user_bcb5a560 field: related.user value: '{{{source.user.id}}}' if: ctx.source?.user?.id != null allow_duplicates: false - append: + tag: append_related_user_f745a124 field: related.user value: '{{{source.user.name}}}' if: ctx.source?.user?.name != null allow_duplicates: false - append: + tag: append_related_user_8c49e24b field: related.user value: '{{{source.user.email}}}' if: ctx.source?.user?.name != null @@ -270,6 +316,7 @@ processors: value: info if: ctx.event?.action != null && !["IMPORT_SSH_PUBLIC_KEY","DELETE_POSIX_ACCOUNT","DELETE_SSH_PUBLIC_KEY","GET_SSH_PUBLIC_KEY","GET_LOGIN_PROFILE","UPDATE_SSH_PUBLIC_KEY"].contains(ctx.event.action) - script: + tag: script_09552c30 lang: painless description: Map Event parameters into key value pair. if: ctx.json?.events?.parameters != null && ctx.json.events.parameters instanceof List @@ -341,14 +388,17 @@ processors: ctx.json.events.parameters = parameters; } - rename: + tag: rename_google_workspace_gcp_USER_EMAIL_to_google_workspace_gcp_user_email_90e329b4 field: google_workspace.gcp.USER_EMAIL target_field: google_workspace.gcp.user_email ignore_missing: true - remove: + tag: remove_e5a80934 field: - json ignore_missing: true - remove: + tag: remove_2a1ed52f if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - google_workspace.ip_address @@ -361,6 +411,7 @@ processors: - google_workspace.id.time ignore_missing: true - script: + tag: script_e2ccbdd7 description: Drops null/empty values recursively. lang: painless source: | @@ -378,10 +429,12 @@ processors: } dropEmptyFields(ctx); - set: + tag: set_event_kind_92954dfa field: event.kind value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/google_workspace/data_stream/gmail/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/gmail/elasticsearch/ingest_pipeline/default.yml index a867c39a8d3..63be136b197 100644 --- a/packages/google_workspace/data_stream/gmail/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/gmail/elasticsearch/ingest_pipeline/default.yml @@ -43,6 +43,7 @@ processors: if: ctx.event?.original != null on_failure: - append: + tag: append_error_message_04014d20 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -112,6 +113,7 @@ processors: } on_failure: - append: + tag: append_error_message_bb64b6b8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' ################################################# @@ -125,6 +127,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_734aabe1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -135,6 +138,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_26e0d7f6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -145,6 +149,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2bbcca3c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -155,6 +160,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2e843f5f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -165,6 +171,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_de2fc4d8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -175,6 +182,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_1f58d569 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -185,6 +193,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8c228191 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -195,6 +204,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_85621e86 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -205,6 +215,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ba520f89 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -215,6 +226,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_0094cd37 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -225,6 +237,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_013be11f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -235,6 +248,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_503601fc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -245,6 +259,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f9f540c2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -255,6 +270,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_645f928f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -265,6 +281,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2bb3701d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -275,6 +292,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8e3f3e68 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -285,6 +303,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9d27956c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -295,6 +314,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_3105ca92 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -305,6 +325,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f40266fc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -315,6 +336,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_875a5c9c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' ################################################# @@ -336,6 +358,7 @@ processors: if: ctx.google_workspace?.gmail?.ip_address != '' on_failure: - append: + tag: append_error_message_b77acb67 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -347,6 +370,7 @@ processors: if: ctx.google_workspace?.gmail?.message_connection_info?.smtp_user_agent_ip != '' on_failure: - append: + tag: append_error_message_66a849ad field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -358,6 +382,7 @@ processors: if: ctx.google_workspace?.gmail?.message_info?.connection_info?.client_ip != '' on_failure: - append: + tag: append_error_message_9925d357 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -369,6 +394,7 @@ processors: if: ctx.google_workspace?.gmail?.message_info?.connection_info?.failed_smtp_out_connect_ip != '' on_failure: - append: + tag: append_error_message_15a1284b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -380,6 +406,7 @@ processors: if: ctx.google_workspace?.gmail?.message_info?.connection_info?.smtp_in_connect_ip != '' on_failure: - append: + tag: append_error_message_9b62d40b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -391,6 +418,7 @@ processors: if: ctx.google_workspace?.gmail?.message_info?.connection_info?.smtp_out_connect_ip != '' on_failure: - append: + tag: append_error_message_6af7fe0b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' ################################################# @@ -404,6 +432,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8996fc3b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -414,6 +443,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_cce22927 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -424,6 +454,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_6b31d36c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -434,6 +465,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_23c3ad94 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -444,6 +476,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_bbb56b5e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -454,6 +487,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a2b349cb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' ################################################# @@ -476,6 +510,7 @@ processors: if: ctx._temp_?.timestamp_usec != null && ctx._temp_.timestamp_usec != '' on_failure: - append: + tag: append_error_message_2d2486d1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -487,6 +522,7 @@ processors: ctx.google_workspace.gmail.message_info.spam_info.classification_timestamp_usec /= 1000; on_failure: - append: + tag: append_error_message_4c9ba31e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -498,6 +534,7 @@ processors: if: ctx.google_workspace?.gmail?.message_info?.spam_info?.classification_timestamp_usec != null && ctx.google_workspace?.gmail.message_info.spam_info.classification_timestamp_usec != '' on_failure: - append: + tag: append_error_message_dd213982 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -509,6 +546,7 @@ processors: ctx.google_workspace.gmail.message_info.structured_policy_log_info.exchange_journal_info.timestamp *= 1000; on_failure: - append: + tag: append_error_message_19e7484b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -520,6 +558,7 @@ processors: if: ctx.google_workspace?.gmail?.message_info?.structured_policy_log_info?.exchange_journal_info?.timestamp != null && ctx.google_workspace.gmail.message_info.structured_policy_log_info.exchange_journal_info.timestamp != '' on_failure: - append: + tag: append_error_message_8bff32a3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' ################################################# @@ -556,6 +595,7 @@ processors: if: ctx.event?.action != '' on_failure: - append: + tag: append_error_message_32466e0d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -575,6 +615,7 @@ processors: pattern: '%{}@%{user.domain}' on_failure: - append: + tag: append_error_message_4c6c1d89 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -587,6 +628,7 @@ processors: ctx.event.duration = ctx.google_workspace.gmail.event_info.elapsed_time_usec * 1000 on_failure: - append: + tag: append_error_message_29dc47ee field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -669,6 +711,7 @@ processors: copy_from: google_workspace.gmail.message_info.connection_info.smtp_tls_version ignore_empty_value: true - foreach: + tag: foreach_google_workspace_gmail_message_info_destination_82398ff4 field: google_workspace.gmail.message_info.destination if: ctx.google_workspace?.gmail?.message_info?.destination instanceof List processor: @@ -703,6 +746,7 @@ processors: pattern: '<%{email.message_id}>' on_failure: - append: + tag: append_error_message_9638fe15 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -720,6 +764,7 @@ processors: ctx.email.content_type = params.get(ctx.google_workspace.gmail.message_info.smime_content_type); on_failure: - append: + tag: append_error_message_97feadc4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' ################################################# @@ -738,6 +783,7 @@ processors: allow_duplicates: false if: ctx.google_workspace?.gmail?.email != null - foreach: + tag: foreach_email_to_address_c2ded133 field: email.to.address if: ctx.email?.to?.address instanceof List processor: @@ -747,6 +793,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - foreach: + tag: foreach_email_from_address_428b8ff0 field: email.from.address if: ctx.email?.from?.address instanceof List processor: @@ -756,6 +803,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - foreach: + tag: foreach_google_workspace_gmail_message_info_triggered_rule_info_92833f9f field: google_workspace.gmail.message_info.triggered_rule_info if: ctx.google_workspace?.gmail?.message_info?.triggered_rule_info instanceof List processor: @@ -765,6 +813,7 @@ processors: value: '{{{_ingest._value.policy_holder_address}}}' allow_duplicates: false - foreach: + tag: foreach_google_workspace_gmail_message_info_connection_info_authenticated_domain_be77471b field: google_workspace.gmail.message_info.connection_info.authenticated_domain if: ctx.google_workspace?.gmail?.message_info?.connection_info?.authenticated_domain instanceof List processor: @@ -798,9 +847,10 @@ processors: allow_duplicates: false if: ctx.google_workspace?.gmail?.message_info?.connection_info?.client_ip != null - foreach: + tag: foreach_google_workspace_gmail_message_info_connection_info_failed_smtp_out_connect_ip_6f03394a field: google_workspace.gmail.message_info.connection_info.failed_smtp_out_connect_ip if: ctx.google_workspace?.gmail?.message_info?.connection_info?.failed_smtp_out_connect_ip instanceof List - processor: + processor: append: field: related.ip tag: append_connection_info_failed_smtp_out_connect_ip_into_related_ip @@ -873,6 +923,7 @@ processors: ignore_missing: true if: ctx.tags == null || !ctx.tags.contains('preserve_duplicate_custom_fields') - foreach: + tag: foreach_google_workspace_gmail_message_info_destination_5158941e field: google_workspace.gmail.message_info.destination if: ctx.google_workspace?.gmail?.message_info?.destination instanceof List && (ctx.tags == null || !ctx.tags.contains('preserve_duplicate_custom_fields')) processor: @@ -919,6 +970,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/google_workspace/data_stream/group_enterprise/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/group_enterprise/elasticsearch/ingest_pipeline/default.yml index 1008fb3553b..29d83913d7f 100644 --- a/packages/google_workspace/data_stream/group_enterprise/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/group_enterprise/elasticsearch/ingest_pipeline/default.yml @@ -13,23 +13,28 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - set: + tag: set_ecs_version_135371bc field: ecs.version value: '8.16.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_90c6475f field: event.original target_field: json on_failure: - append: + tag: append_error_message_45b808be field: error.message value: '{{{_ingest.on_failure_message}}}' - drop: @@ -37,6 +42,7 @@ processors: description: Discard events that are missing the target during the split operation and are subsequently returned as the root object. tag: drop_empty_events - date: + tag: date_json_id_time_9350ecd5 field: json.id.time if: ctx.json?.id?.time != null && ctx.json.id.time != '' formats: @@ -48,13 +54,16 @@ processors: - yyyy/MM/dd HH:mm:ss z on_failure: - append: + tag: append_error_message_18a82a92 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_time_01ac69ca field: google_workspace.id.time copy_from: '@timestamp' ignore_empty_value: true - fingerprint: + tag: fingerprint_52623c07 description: Hashes the ID object and uses it as the document id to avoid duplicate events. fields: - json.id @@ -62,73 +71,90 @@ processors: target_field: _id ignore_missing: true - rename: + tag: rename_json_events_name_to_event_action_75ca640e field: json.events.name target_field: event.action ignore_missing: true - set: + tag: set_google_workspace_event_name_69e4d7fd field: google_workspace.event.name copy_from: event.action ignore_empty_value: true - rename: + tag: rename_json_id_applicationName_to_event_provider_fffca35d field: json.id.applicationName target_field: event.provider ignore_missing: true - set: + tag: set_google_workspace_id_application_name_8030da4a field: google_workspace.id.application_name copy_from: event.provider ignore_empty_value: true - convert: + tag: convert_json_id_uniqueQualifier_to_event_id_0239362e field: json.id.uniqueQualifier target_field: event.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_ab6a6ac3 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_unique_qualifier_8c91fe06 field: google_workspace.id.unique_qualifier copy_from: event.id ignore_empty_value: true - rename: + tag: rename_json_actor_email_to_source_user_email_4ee5e9ab field: json.actor.email target_field: source.user.email ignore_missing: true - set: + tag: set_google_workspace_actor_email_9cd77425 field: google_workspace.actor.email copy_from: source.user.email ignore_empty_value: true - set: + tag: set_user_email_371886fa field: user.email copy_from: source.user.email ignore_empty_value: true - convert: + tag: convert_json_actor_profileId_to_source_user_id_e10534f1 field: json.actor.profileId target_field: source.user.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_dc10d8d6 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_actor_profile_id_6b7a4346 field: google_workspace.actor.profile.id copy_from: source.user.id ignore_empty_value: true - convert: + tag: convert_json_ipAddress_to_source_ip_6cbf48a8 field: json.ipAddress target_field: source.ip type: ip ignore_missing: true on_failure: - append: + tag: append_error_message_79a0c21d field: error.message value: '{{{_ingest.on_failure_message}}}' - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -137,55 +163,68 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - set: + tag: set_google_workspace_ip_address_b61dbbba field: google_workspace.ip_address copy_from: source.ip ignore_empty_value: true - rename: + tag: rename_json_kind_to_google_workspace_kind_8c6d9f29 field: json.kind target_field: google_workspace.kind ignore_missing: true - rename: + tag: rename_json_etag_to_google_workspace_etag_4bd4a803 field: json.etag target_field: google_workspace.etag ignore_missing: true - convert: + tag: convert_json_id_customerId_to_organization_id_ffd90a33 field: json.id.customerId target_field: organization.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_5aacea86 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_customer_id_46a3415a field: google_workspace.id.customer.id copy_from: organization.id ignore_empty_value: true - rename: + tag: rename_json_actor_callerType_to_google_workspace_actor_type_308eeffc field: json.actor.callerType target_field: google_workspace.actor.type ignore_missing: true - rename: + tag: rename_json_actor_key_to_google_workspace_actor_key_3f392d49 field: json.actor.key target_field: google_workspace.actor.key ignore_missing: true - rename: + tag: rename_json_ownerDomain_to_google_workspace_organization_domain_6580cd7d field: json.ownerDomain target_field: google_workspace.organization.domain ignore_missing: true - rename: + tag: rename_json_events_type_to_google_workspace_event_type_e73f1e18 field: json.events.type target_field: google_workspace.event.type ignore_missing: true - set: + tag: set_user_id_f89889ac field: user.id copy_from: source.user.id ignore_empty_value: true @@ -232,6 +271,7 @@ processors: value: deletion if: ctx.event?.action != null && ["delete_group", "delete_namespace", "remove_info_setting", "remove_security_setting", "remove_service_account_permission", "remove_member", "remove_member_role", "remove_membership_expiry", "revoke_invitation"].contains(ctx.event.action) - script: + tag: script_a3a2cdbf lang: painless description: Extract User name and User domain from Email. if: ctx.source?.user?.email != null && ctx.source.user.email.contains("@") @@ -248,36 +288,43 @@ processors: ctx.user.domain = splitmail[1]; ctx.source.user.domain = splitmail[1]; - append: + tag: append_related_hosts_23e97a1f field: related.hosts value: '{{{user.domain}}}' if: ctx.user?.domain != null allow_duplicates: false - append: + tag: append_related_hosts_ea7c8eda field: related.hosts value: '{{{google_workspace.organization.domain}}}' if: ctx.google_workspace?.organization?.domain != null allow_duplicates: false - append: + tag: append_related_ip_8121c591 field: related.ip value: '{{{source.ip}}}' if: ctx.source?.ip != null allow_duplicates: false - append: + tag: append_related_user_bcb5a560 field: related.user value: '{{{source.user.id}}}' if: ctx.source?.user?.id != null allow_duplicates: false - append: + tag: append_related_user_f745a124 field: related.user value: '{{{source.user.name}}}' if: ctx.source?.user?.name != null allow_duplicates: false - append: + tag: append_related_user_8c49e24b field: related.user value: '{{{source.user.email}}}' if: ctx.source?.user?.name != null allow_duplicates: false - script: + tag: script_6aa7ff57 lang: painless description: Map Event parameters into key value pair. if: ctx.json?.events?.parameters != null && ctx.json.events.parameters instanceof List @@ -349,26 +396,32 @@ processors: ctx.json.events.parameters = parameters; } - rename: + tag: rename_google_workspace_group_enterprise_group_id_to_google_workspace_group_enterprise_group_id_e2e6d661 field: google_workspace.group_enterprise.group_id target_field: google_workspace.group_enterprise.group.id ignore_missing: true - set: + tag: set_group_id_b2ab2c93 field: group.id copy_from: google_workspace.group_enterprise.group.id ignore_empty_value: true - rename: + tag: rename_google_workspace_group_enterprise_member_id_to_google_workspace_group_enterprise_member_id_ded16a57 field: google_workspace.group_enterprise.member_id target_field: google_workspace.group_enterprise.member.id ignore_missing: true - rename: + tag: rename_google_workspace_group_enterprise_member_role_to_google_workspace_group_enterprise_member_role_311ff839 field: google_workspace.group_enterprise.member_role target_field: google_workspace.group_enterprise.member.role ignore_missing: true - rename: + tag: rename_google_workspace_group_enterprise_member_type_to_google_workspace_group_enterprise_member_type_0a9d7675 field: google_workspace.group_enterprise.member_type target_field: google_workspace.group_enterprise.member.type ignore_missing: true - date: + tag: date_google_workspace_group_enterprise_membership_expiry_to_google_workspace_group_enterprise_membership_expiry_d5ee1e35 field: google_workspace.group_enterprise.membership_expiry target_field: google_workspace.group_enterprise.membership_expiry if: ctx.google_workspace?.group_enterprise?.membership_expiry != null && ctx.google_workspace.group_enterprise.membership_expiry != '' @@ -381,21 +434,26 @@ processors: - yyyy/MM/dd HH:mm:ss z on_failure: - append: + tag: append_error_message_49f1ca8a field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_google_workspace_group_enterprise_security_setting_to_google_workspace_group_enterprise_security_setting_value_76093d15 field: google_workspace.group_enterprise.security_setting target_field: google_workspace.group_enterprise.security_setting.value ignore_missing: true - rename: + tag: rename_google_workspace_group_enterprise_security_setting_state_to_google_workspace_group_enterprise_security_setting_state_33535cbf field: google_workspace.group_enterprise.security_setting_state target_field: google_workspace.group_enterprise.security_setting.state ignore_missing: true - remove: + tag: remove_e5a80934 field: - json ignore_missing: true - remove: + tag: remove_009bf161 if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - google_workspace.ip_address @@ -409,6 +467,7 @@ processors: - google_workspace.id.time ignore_missing: true - script: + tag: script_e2ccbdd7 description: Drops null/empty values recursively. lang: painless source: | @@ -426,10 +485,12 @@ processors: } dropEmptyFields(ctx); - set: + tag: set_event_kind_92954dfa field: event.kind value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/google_workspace/data_stream/groups/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/groups/elasticsearch/ingest_pipeline/default.yml index 5a04403526c..ad19fd8952a 100644 --- a/packages/google_workspace/data_stream/groups/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/groups/elasticsearch/ingest_pipeline/default.yml @@ -13,28 +13,35 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - set: + tag: set_ecs_version_135371bc field: ecs.version value: '8.16.0' - append: + tag: append_event_category_576de059 field: event.category value: iam - append: + tag: append_event_type_282e488b field: event.type value: group - set: + tag: set_event_kind_de80643c field: event.kind value: event - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - drop: @@ -42,6 +49,7 @@ processors: description: Discard events that are missing the target during the split operation and are subsequently returned as the root object. tag: drop_empty_events - date: + tag: date_json_id_time_dc34ff3c field: json.id.time if: ctx.json?.id?.time != null && ctx.json.id.time != '' timezone: UTC @@ -53,9 +61,11 @@ processors: - yyyy/MM/dd HH:mm:ss z on_failure: - append: + tag: append_error_message_a48f8d5f field: error.message value: "{{{_ingest.on_failure_message}}}" - fingerprint: + tag: fingerprint_567b61c4 description: Hashes the ID object and uses it as the document id to avoid duplicate events. fields: - json.id @@ -64,37 +74,45 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_events_name_to_event_action_75ca640e field: json.events.name target_field: event.action ignore_missing: true - rename: + tag: rename_json_id_applicationName_to_event_provider_fffca35d field: json.id.applicationName target_field: event.provider ignore_missing: true - convert: + tag: convert_json_id_uniqueQualifier_to_event_id_0664525d field: json.id.uniqueQualifier target_field: event.id type: string ignore_missing: true - rename: + tag: rename_json_actor_email_to_source_user_email_4ee5e9ab field: json.actor.email target_field: source.user.email ignore_missing: true - set: + tag: set_user_email_f8429233 field: user.email copy_from: source.user.email ignore_failure: true - convert: + tag: convert_json_actor_profileId_to_source_user_id_edfcffb4 field: json.actor.profileId target_field: source.user.id type: string ignore_missing: true - convert: + tag: convert_json_ipAddress_to_source_ip_aabdaf8f field: json.ipAddress target_field: source.ip type: ip ignore_missing: true - rename: + tag: rename_json_kind_to_google_workspace_kind_8c6d9f29 field: json.kind target_field: google_workspace.kind ignore_missing: true @@ -106,29 +124,36 @@ processors: tag: convert_customer_id_to_string on_failure: - append: + tag: append_error_message_044cfe8e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_actor_callerType_to_google_workspace_actor_type_308eeffc field: json.actor.callerType target_field: google_workspace.actor.type ignore_missing: true - rename: + tag: rename_json_actor_key_to_google_workspace_actor_key_3f392d49 field: json.actor.key target_field: google_workspace.actor.key ignore_missing: true - rename: + tag: rename_json_ownerDomain_to_google_workspace_organization_domain_6580cd7d field: json.ownerDomain target_field: google_workspace.organization.domain ignore_missing: true - rename: + tag: rename_json_events_type_to_google_workspace_event_type_e73f1e18 field: json.events.type target_field: google_workspace.event.type ignore_missing: true - set: + tag: set_user_id_64f70b73 field: user.id copy_from: source.user.id if: ctx?.source?.user?.id != null - script: + tag: script_8d756e1a lang: painless if: 'ctx?.source?.user?.email != null && ctx?.source?.user?.email.contains("@")' source: > @@ -144,40 +169,49 @@ processors: ctx.user.domain = splitmail[1]; ctx.source.user.domain = splitmail[1]; - append: + tag: append_related_ip_549fa858 field: related.ip value: "{{{source.ip}}}" if: ctx?.source?.ip != null allow_duplicates: false - append: + tag: append_related_user_8302ba1f field: related.user value: "{{{source.user.name}}}" if: ctx?.source?.user?.name != null allow_duplicates: false - - append: + - append: + tag: append_event_type_900ff825 field: event.type value: change if: '["change_basic_setting", "change_identity_setting", "change_info_setting", "change_new_members_restrictions_setting", "change_post_replies_setting", "change_spam_moderation_setting", "change_topic_setting", "change_acl_permission", "approve_join_request", "join"].contains(ctx?.event?.action)' - - append: + - append: + tag: append_event_category_c078fa55 field: event.category value: configuration if: '["change_basic_setting", "change_identity_setting", "change_info_setting", "change_new_members_restrictions_setting", "change_post_replies_setting", "change_spam_moderation_setting", "change_topic_setting", "add_info_setting", "remove_info_setting"].contains(ctx?.event?.action)' - - append: + - append: + tag: append_event_type_a6341fe0 field: event.type value: info if: '["accept_invitation", "request_to_join", "ban_user_with_moderation", "revoke_invitation", "invite_user", "reject_join_request", "reinvite_user", "moderate_message", "always_post_from_user"].contains(ctx?.event?.action)' - - append: + - append: + tag: append_event_type_bf3c9f50 field: event.type value: user if: '["accept_invitation", "approve_join_request", "join", "request_to_join", "ban_user_with_moderation", "revoke_invitation", "invite_user", "reject_join_request", "reinvite_user", "add_user", "remove_user"].contains(ctx?.event?.action)' - - append: + - append: + tag: append_event_type_b7376da7 field: event.type value: creation if: '["create_group", "add_info_setting", "add_user"].contains(ctx?.event?.action)' - - append: + - append: + tag: append_event_type_b440f4a5 field: event.type value: deletion if: '["delete_group", "remove_info_setting", "remove_user"].contains(ctx?.event?.action)' - script: + tag: script_fd653f6d lang: painless if: 'ctx?.json?.events?.parameters != null && ctx?.json?.events?.parameters instanceof List' source: > @@ -196,70 +230,87 @@ processors: } } - rename: + tag: rename_google_workspace_groups_group_email_to_google_workspace_groups_email_678a6528 field: google_workspace.groups.group_email target_field: google_workspace.groups.email ignore_missing: true - rename: + tag: rename_google_workspace_groups_new_value_repeated_to_google_workspace_groups_new_value_3c4cf20f field: google_workspace.groups.new_value_repeated target_field: google_workspace.groups.new_value ignore_missing: true - rename: + tag: rename_google_workspace_groups_old_value_repeated_to_google_workspace_groups_old_value_79a73143 field: google_workspace.groups.old_value_repeated target_field: google_workspace.groups.old_value ignore_missing: true - rename: + tag: rename_google_workspace_groups_user_email_to_google_workspace_groups_member_email_271af844 field: google_workspace.groups.user_email target_field: google_workspace.groups.member.email ignore_missing: true - rename: + tag: rename_google_workspace_groups_basic_setting_to_google_workspace_groups_setting_0ee9e98d field: google_workspace.groups.basic_setting target_field: google_workspace.groups.setting ignore_missing: true - rename: + tag: rename_google_workspace_groups_identity_setting_to_google_workspace_groups_setting_7a5d6089 field: google_workspace.groups.identity_setting target_field: google_workspace.groups.setting ignore_missing: true - rename: + tag: rename_google_workspace_groups_info_setting_to_google_workspace_groups_setting_f2656dbf field: google_workspace.groups.info_setting target_field: google_workspace.groups.setting ignore_missing: true - rename: + tag: rename_google_workspace_groups_new_members_restrictions_setting_to_google_workspace_groups_setting_73635047 field: google_workspace.groups.new_members_restrictions_setting target_field: google_workspace.groups.setting ignore_missing: true - rename: + tag: rename_google_workspace_groups_post_replies_setting_to_google_workspace_groups_setting_1e2f4cae field: google_workspace.groups.post_replies_setting target_field: google_workspace.groups.setting ignore_missing: true - rename: + tag: rename_google_workspace_groups_spam_moderation_setting_to_google_workspace_groups_setting_ff1c2bd1 field: google_workspace.groups.spam_moderation_setting target_field: google_workspace.groups.setting ignore_missing: true - rename: + tag: rename_google_workspace_groups_topic_setting_to_google_workspace_groups_setting_1fd57144 field: google_workspace.groups.topic_setting target_field: google_workspace.groups.setting ignore_missing: true - rename: + tag: rename_google_workspace_groups_message_id_to_google_workspace_groups_message_id_168dd0a9 field: google_workspace.groups.message_id target_field: google_workspace.groups.message.id ignore_missing: true - rename: + tag: rename_google_workspace_groups_message_moderation_action_to_google_workspace_groups_message_moderation_action_3b810d61 field: google_workspace.groups.message_moderation_action target_field: google_workspace.groups.message.moderation_action ignore_missing: true - rename: + tag: rename_google_workspace_groups_member_role_to_google_workspace_groups_member_role_9ef724bb field: google_workspace.groups.member_role target_field: google_workspace.groups.member.role ignore_missing: true - set: + tag: set_event_outcome_8b4f986f field: event.outcome value: failure if: 'ctx?.google_workspace?.groups?.status == "failed"' - set: + tag: set_event_outcome_fb386d50 field: event.outcome value: success if: 'ctx?.google_workspace?.groups?.status == "success"' - script: + tag: script_085f346b lang: painless if: 'ctx?.google_workspace?.groups?.email != null && ctx?.google_workspace?.groups?.email.contains("@")' source: > @@ -273,6 +324,7 @@ processors: ctx.group.name = splitmail[0]; ctx.group.domain = splitmail[1]; - script: + tag: script_769d8806 lang: painless if: 'ctx?.google_workspace?.groups?.member?.email != null && ctx?.google_workspace?.groups?.member?.email.contains("@")' source: > @@ -297,18 +349,22 @@ processors: ctx.user.target.domain = splitmail[1]; ctx.user.target.email = ctx.google_workspace.groups.member.email; - set: + tag: set_user_target_group_name_fa952c70 field: user.target.group.name copy_from: group.name if: ctx?.group?.name != null - set: + tag: set_user_target_group_domain_09d0d95f field: user.target.group.domain copy_from: group.domain if: ctx?.group?.domain != null - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -317,14 +373,17 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - remove: + tag: remove_json_29cdffdc field: json ignore_missing: true on_failure: diff --git a/packages/google_workspace/data_stream/keep/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/keep/elasticsearch/ingest_pipeline/default.yml index 10072a75a86..04efdadf7fb 100644 --- a/packages/google_workspace/data_stream/keep/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/keep/elasticsearch/ingest_pipeline/default.yml @@ -48,6 +48,7 @@ processors: if: ctx.event?.original != null on_failure: - append: + tag: append_error_message_04014d20 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -79,9 +80,11 @@ processors: } on_failure: - append: + tag: append_error_message_c6c5725c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: + tag: remove_json_events_parameters_2eab3501 field: json.events.parameters ignore_missing: true - rename: @@ -125,6 +128,7 @@ processors: } on_failure: - append: + tag: append_error_message_a4264f36 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -132,7 +136,8 @@ processors: tag: rename_json_kind target_field: google_workspace.kind ignore_missing: true - - rename: + - rename: + tag: rename_json_ownerDomain_to_google_workspace_organization_domain_6580cd7d field: json.ownerDomain target_field: google_workspace.organization.domain ignore_missing: true @@ -168,6 +173,7 @@ processors: pattern: '%{}@%{source.user.domain}' on_failure: - append: + tag: append_error_message_4450a0e8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -220,6 +226,7 @@ processors: if: ctx.json?.id?.time != null && ctx.json.id.time != '' on_failure: - append: + tag: append_error_message_de0bed15 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -246,6 +253,7 @@ processors: if: ctx.json?.ipAddress != '' on_failure: - append: + tag: append_error_message_4cb1517b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -254,10 +262,12 @@ processors: copy_from: google_workspace.ip_address ignore_empty_value: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 field: source.ip database_file: GeoLite2-ASN.mmdb target_field: source.as @@ -266,10 +276,12 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true @@ -292,6 +304,7 @@ processors: if: ctx.event?.action != '' on_failure: - append: + tag: append_error_message_32466e0d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -354,6 +367,7 @@ processors: drop(ctx); on_failure: - append: + tag: append_error_message_0886867f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -362,6 +376,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/google_workspace/data_stream/login/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/login/elasticsearch/ingest_pipeline/default.yml index f8ac1decc2f..5727e7d6a75 100644 --- a/packages/google_workspace/data_stream/login/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/login/elasticsearch/ingest_pipeline/default.yml @@ -13,19 +13,23 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - set: + tag: set_ecs_version_135371bc field: ecs.version value: '8.16.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - drop: @@ -33,9 +37,11 @@ processors: description: Discard events that are missing the target during the split operation and are subsequently returned as the root object. tag: drop_empty_events - set: + tag: set_event_kind_de80643c field: event.kind value: event - date: + tag: date_json_id_time_dc34ff3c field: json.id.time if: ctx.json?.id?.time != null && ctx.json.id.time != '' timezone: UTC @@ -47,9 +53,11 @@ processors: - yyyy/MM/dd HH:mm:ss z on_failure: - append: + tag: append_error_message_a48f8d5f field: error.message value: "{{{_ingest.on_failure_message}}}" - fingerprint: + tag: fingerprint_567b61c4 description: Hashes the ID object and uses it as the document id to avoid duplicate events. fields: - json.id @@ -58,37 +66,45 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_events_name_to_event_action_75ca640e field: json.events.name target_field: event.action ignore_missing: true - rename: + tag: rename_json_id_applicationName_to_event_provider_fffca35d field: json.id.applicationName target_field: event.provider ignore_missing: true - convert: + tag: convert_json_id_uniqueQualifier_to_event_id_0664525d field: json.id.uniqueQualifier target_field: event.id type: string ignore_missing: true - rename: + tag: rename_json_actor_email_to_source_user_email_4ee5e9ab field: json.actor.email target_field: source.user.email ignore_missing: true - set: + tag: set_user_email_f8429233 field: user.email copy_from: source.user.email ignore_failure: true - convert: + tag: convert_json_actor_profileId_to_source_user_id_edfcffb4 field: json.actor.profileId target_field: source.user.id type: string ignore_missing: true - convert: + tag: convert_json_ipAddress_to_source_ip_aabdaf8f field: json.ipAddress target_field: source.ip type: ip ignore_missing: true - rename: + tag: rename_json_kind_to_google_workspace_kind_8c6d9f29 field: json.kind target_field: google_workspace.kind ignore_missing: true @@ -100,25 +116,31 @@ processors: tag: convert_customer_id_to_string on_failure: - append: + tag: append_error_message_044cfe8e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_actor_callerType_to_google_workspace_actor_type_308eeffc field: json.actor.callerType target_field: google_workspace.actor.type ignore_missing: true - rename: + tag: rename_json_actor_key_to_google_workspace_actor_key_3f392d49 field: json.actor.key target_field: google_workspace.actor.key ignore_missing: true - rename: + tag: rename_json_ownerDomain_to_google_workspace_organization_domain_6580cd7d field: json.ownerDomain target_field: google_workspace.organization.domain ignore_missing: true - rename: + tag: rename_json_events_type_to_google_workspace_event_type_e73f1e18 field: json.events.type target_field: google_workspace.event.type ignore_missing: true - script: + tag: script_8d756e1a lang: painless if: 'ctx?.source?.user?.email != null && ctx?.source?.user?.email.contains("@")' source: > @@ -134,48 +156,59 @@ processors: ctx.user.domain = splitmail[1]; ctx.source.user.domain = splitmail[1]; - set: + tag: set_user_id_64f70b73 field: user.id copy_from: source.user.id if: ctx?.source?.user?.id != null - append: + tag: append_related_ip_549fa858 field: related.ip value: "{{{source.ip}}}" if: ctx?.source?.ip != null allow_duplicates: false - append: + tag: append_related_user_8302ba1f field: related.user value: "{{{source.user.name}}}" if: ctx?.source?.user?.name != null allow_duplicates: false - append: + tag: append_event_category_0ad925bf field: event.category value: authentication if: '["suspicious_login","suspicious_login_less_secure_app","suspicious_programmatic_login","user_signed_out_due_to_suspicious_session_cookie","login_failure","login_challenge","login_verification","logout","login_success"].contains(ctx?.event?.action)' - append: + tag: append_event_category_862a8460 field: event.category value: iam if: '["account_disabled_password_leak","2sv_disable","2sv_enroll","account_disabled_generic","account_disabled_spamming_through_relay","account_disabled_spamming","account_disabled_hijacked","titanium_enroll","titanium_unenroll","gov_attack_warning","passkey_enrolled","passkey_removed"].contains(ctx.event?.action)' - append: + tag: append_event_category_55dc9de0 field: event.category value: configuration if: '["password_edit","recovery_email_edit","recovery_phone_edit","recovery_secret_qa_edit","blocked_sender","email_forwarding_out_of_domain","risky_sensitive_action_allowed","risky_sensitive_action_blocked"].contains(ctx?.event?.action)' - append: + tag: append_event_category_7fdb64db field: event.category value: session if: '["login_failure","login_success","logout"].contains(ctx?.event?.action)' - append: + tag: append_event_type_700c8f91 field: event.type value: start if: '["login_failure","login_success"].contains(ctx?.event?.action)' - append: + tag: append_event_type_c5b41421 field: event.type value: end if: '["logout","user_signed_out_due_to_suspicious_session_cookie"].contains(ctx?.event?.action)' - append: + tag: append_event_type_19753b1b field: event.type value: user if: '["account_disabled_password_leak","account_disabled_generic","account_disabled_spamming_through_relay","account_disabled_spamming","account_disabled_hijacked","gov_attack_warning"].contains(ctx?.event?.action)' - append: + tag: append_event_type_6961815c field: event.type value: change if: '["password_edit","recovery_email_edit","recovery_phone_edit","recovery_secret_qa_edit","account_disabled_generic","account_disabled_spamming_through_relay","account_disabled_spamming","account_disabled_hijacked"].contains(ctx?.event?.action)' @@ -190,22 +223,27 @@ processors: value: deletion if: ctx.event?.action == 'passkey_removed' - append: + tag: append_event_type_93616ccd field: event.type value: info if: '["2sv_disable","2sv_enroll","suspicious_login","suspicious_login_less_secure_app","suspicious_programmatic_login","titanium_enroll","titanium_unenroll","blocked_sender","email_forwarding_out_of_domain","login_challenge","login_verification","risky_sensitive_action_allowed","risky_sensitive_action_blocked","account_disabled_password_leak"].contains(ctx?.event?.action)' - set: + tag: set_event_kind_dab39c46 field: event.kind value: alert if: '["gov_attack_warning"].contains(ctx?.event?.action)' - set: + tag: set_event_outcome_c64393de field: event.outcome value: failure if: 'ctx?.event?.action == "login_failure"' - set: + tag: set_event_outcome_2f7e52b2 field: event.outcome value: success if: 'ctx?.event?.action == "login_success"' - script: + tag: script_8c31eaf3 lang: painless if: 'ctx?.json?.events?.parameters != null && ctx?.json?.events?.parameters instanceof List' source: > @@ -230,48 +268,58 @@ processors: } } - rename: + tag: rename_json_networkInfo_ipAsn_to_google_workspace_login_network_info_ip_asn_47e6b506 field: json.networkInfo.ipAsn target_field: google_workspace.login.network_info.ip_asn ignore_missing: true - rename: + tag: rename_json_networkInfo_regionCode_to_google_workspace_login_network_info_region_code_21f4d5a4 field: json.networkInfo.regionCode target_field: google_workspace.login.network_info.region_code ignore_missing: true - rename: + tag: rename_json_networkInfo_subdivisionCode_to_google_workspace_login_network_info_subdivision_code_11b8939c field: json.networkInfo.subdivisionCode target_field: google_workspace.login.network_info.subdivision_code ignore_missing: true - rename: + tag: rename_json_resourceIds_to_google_workspace_login_resource_ids_3ee11581 field: json.resourceIds target_field: google_workspace.login.resource_ids ignore_missing: true - convert: + tag: convert_google_workspace_login_timestamp_to_google_workspace_login_timestamp_ef394597 field: google_workspace.login.timestamp target_field: google_workspace.login.timestamp type: long ignore_missing: true - script: + tag: script_5351eb5a lang: painless if: ctx?.google_workspace?.login?.timestamp != null source: > ctx._temp_ = new HashMap(); ctx._temp_.start = ctx?.google_workspace?.login.timestamp / 1000; - date: + tag: date__temp__start_to_event_start_66e664ee field: _temp_.start target_field: event.start timezone: UTC formats: - UNIX_MS if: ctx?._temp_?.start != null - - set: + - set: + tag: set_event_outcome_dd2f9147 field: event.outcome value: success if: ctx.event?.outcome == null && ctx.google_workspace?.login?.challenge_status?.toLowerCase()?.contains('passed') == true - - set: + - set: + tag: set_event_outcome_40353fcc field: event.outcome value: failure if: 'ctx.google_workspace?.login?.challenge_status != null && ctx.event?.outcome == null' - script: + tag: script_f160a7de lang: painless if: 'ctx?.google_workspace?.login?.affected_email_address != null && ctx?.google_workspace?.login?.affected_email_address.contains("@")' source: > @@ -295,10 +343,12 @@ processors: ctx.user.target.domain = splitmail[1]; ctx.related.user.add(splitmail[0]); - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -307,15 +357,18 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - remove: - field: + tag: remove_86fcb9a0 + field: - json - _temp_ ignore_missing: true diff --git a/packages/google_workspace/data_stream/meet/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/meet/elasticsearch/ingest_pipeline/default.yml index 5e2b6aa6e8a..d40140645f2 100644 --- a/packages/google_workspace/data_stream/meet/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/meet/elasticsearch/ingest_pipeline/default.yml @@ -48,6 +48,7 @@ processors: if: ctx.event?.original != null on_failure: - append: + tag: append_error_message_04014d20 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -79,9 +80,11 @@ processors: } on_failure: - append: + tag: append_error_message_c4c0b264 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: + tag: remove_json_events_parameters_2eab3501 field: json.events.parameters ignore_missing: true - rename: @@ -154,6 +157,7 @@ processors: } on_failure: - append: + tag: append_error_message_18c02496 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -161,7 +165,8 @@ processors: tag: rename_json_kind target_field: google_workspace.kind ignore_missing: true - - rename: + - rename: + tag: rename_json_ownerDomain_to_google_workspace_organization_domain_6580cd7d field: json.ownerDomain target_field: google_workspace.organization.domain ignore_missing: true @@ -203,6 +208,7 @@ processors: pattern: '%{}@%{source.user.domain}' on_failure: - append: + tag: append_error_message_4450a0e8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -255,6 +261,7 @@ processors: if: ctx.json?.id?.time != null && ctx.json.id.time != '' on_failure: - append: + tag: append_error_message_de0bed15 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -281,6 +288,7 @@ processors: if: ctx.json?.ipAddress != '' on_failure: - append: + tag: append_error_message_4cb1517b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -289,10 +297,12 @@ processors: copy_from: google_workspace.ip_address ignore_empty_value: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 field: source.ip database_file: GeoLite2-ASN.mmdb target_field: source.as @@ -301,10 +311,12 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true @@ -327,6 +339,7 @@ processors: if: ctx.event?.action != '' on_failure: - append: + tag: append_error_message_32466e0d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -340,6 +353,7 @@ processors: if: ctx.google_workspace?.meet?.action_time != null && ctx.google_workspace.meet.action_time != '' on_failure: - append: + tag: append_error_message_7d761361 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -355,6 +369,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_06d18e99 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -365,6 +380,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_411fd89a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -375,6 +391,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2029038b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -385,6 +402,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9cd5abde field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -395,6 +413,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a7ffc60f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -405,6 +424,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a800e940 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -415,6 +435,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_145f883d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -435,6 +456,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2584a5f1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -458,6 +480,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_7276274b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -484,6 +507,7 @@ processors: if: ctx.google_workspace?.meet?.ip_address != '' on_failure: - append: + tag: append_error_message_866300f9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -493,10 +517,12 @@ processors: allow_duplicates: false if: ctx.google_workspace?.meet?.endpoint?.ip_address != null - geoip: + tag: geoip_host_ip_to_host_geo_d8a29594 field: host.ip target_field: host.geo ignore_missing: true - geoip: + tag: geoip_host_ip_to_host_as_ca603f73 field: host.ip database_file: GeoLite2-ASN.mmdb target_field: host.as @@ -505,10 +531,12 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_host_as_asn_to_host_as_number_7622e953 field: host.as.asn target_field: host.as.number ignore_missing: true - rename: + tag: rename_host_as_organization_name_to_host_as_organization_name_7b13ca3d field: host.as.organization_name target_field: host.as.organization.name ignore_missing: true @@ -520,6 +548,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ac58bf67 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -545,6 +574,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5a1293de field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -555,6 +585,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_694c88c5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -565,6 +596,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9caf9308 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -575,6 +607,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5c542fb4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -585,6 +618,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_64eed17b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -595,6 +629,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f749593c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -605,6 +640,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_12f9bff5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -629,6 +665,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d1541d2d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -639,6 +676,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c586caf8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -649,6 +687,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_316582ec field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -659,6 +698,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_b80119ee field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -669,6 +709,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_58dc2c63 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -679,6 +720,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_133dbee8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -689,6 +731,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f537f9e4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -699,6 +742,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_18e30dcb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -709,6 +753,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_edf8bdee field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -719,6 +764,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c1aaa35a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -729,6 +775,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f7e497d8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -739,6 +786,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a0deca35 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -749,6 +797,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_6ddf3c2e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -759,6 +808,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4ad0639a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -769,6 +819,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9e9f33c4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -789,6 +840,7 @@ processors: if: ctx.event?.start != null && ctx.event.start != '' on_failure: - append: + tag: append_error_message_1f290f15 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -799,6 +851,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5bfa1b98 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -809,6 +862,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_0e841e44 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -819,6 +873,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c613f9b2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -829,6 +884,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8daa062d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -839,6 +895,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_42cdf828 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -849,6 +906,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_7b88baa0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -859,6 +917,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_72d26f45 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -869,6 +928,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_badd0416 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -879,6 +939,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e5482d9a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -889,6 +950,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_48268e8c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -899,6 +961,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_fdf62e7f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -909,6 +972,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f1fa4936 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -919,6 +983,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4c89def6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -940,6 +1005,7 @@ processors: if: ctx.google_workspace?.meet?.livestream_private_ip_address != '' on_failure: - append: + tag: append_error_message_46b50e9e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -956,6 +1022,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d59a2485 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -980,6 +1047,7 @@ processors: pattern: '%{}@%{user.target.domain}' on_failure: - append: + tag: append_error_message_6f7e470a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1000,6 +1068,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_59cb2fa4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -1021,6 +1090,7 @@ processors: allow_duplicates: false if: ctx.google_workspace?.meet?.organizer_email != null - foreach: + tag: foreach_google_workspace_meet_target_display_names_1b140350 field: google_workspace.meet.target.display_names if: ctx.google_workspace?.meet?.target?.display_names instanceof List processor: @@ -1042,6 +1112,7 @@ processors: allow_duplicates: false if: ctx.google_workspace?.meet?.endpoint?.identifier != null && ctx.google_workspace?.meet?.endpoint?.identifier_type == "email_address" - foreach: + tag: foreach_host_ip_e0593a1f field: host.ip if: ctx.host?.ip instanceof List processor: @@ -1159,6 +1230,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/google_workspace/data_stream/rules/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/rules/elasticsearch/ingest_pipeline/default.yml index 5687fdfa35f..94fb18bae61 100644 --- a/packages/google_workspace/data_stream/rules/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/rules/elasticsearch/ingest_pipeline/default.yml @@ -13,19 +13,23 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - set: + tag: set_ecs_version_135371bc field: ecs.version value: '8.16.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - drop: @@ -33,6 +37,7 @@ processors: description: Discard events that are missing the target during the split operation and are subsequently returned as the root object. tag: drop_empty_events - date: + tag: date_json_id_time_9350ecd5 field: json.id.time if: ctx.json?.id?.time != null && ctx.json.id.time != '' formats: @@ -44,13 +49,16 @@ processors: - yyyy/MM/dd HH:mm:ss z on_failure: - append: + tag: append_error_message_18a82a92 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_time_01ac69ca field: google_workspace.id.time copy_from: '@timestamp' ignore_empty_value: true - fingerprint: + tag: fingerprint_567b61c4 description: Hashes the ID object and uses it as the document id to avoid duplicate events. fields: - json.id @@ -59,114 +67,141 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_events_name_to_event_action_75ca640e field: json.events.name target_field: event.action ignore_missing: true - set: + tag: set_google_workspace_event_name_b5ff2dc4 field: google_workspace.event.name copy_from: event.action ignore_failure: true - rename: + tag: rename_json_id_applicationName_to_event_provider_fffca35d field: json.id.applicationName target_field: event.provider ignore_missing: true - set: + tag: set_google_workspace_id_application_name_6d6b3503 field: google_workspace.id.application_name copy_from: event.provider ignore_failure: true - convert: + tag: convert_json_id_uniqueQualifier_to_event_id_0239362e field: json.id.uniqueQualifier target_field: event.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_ab6a6ac3 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_unique_qualifier_cf38ca4f field: google_workspace.id.unique_qualifier copy_from: event.id ignore_failure: true - rename: + tag: rename_json_actor_email_to_source_user_email_4ee5e9ab field: json.actor.email target_field: source.user.email ignore_missing: true - set: + tag: set_google_workspace_actor_email_9749234c field: google_workspace.actor.email copy_from: source.user.email ignore_failure: true - set: + tag: set_user_email_f8429233 field: user.email copy_from: source.user.email ignore_failure: true - convert: + tag: convert_json_actor_profileId_to_source_user_id_e10534f1 field: json.actor.profileId target_field: source.user.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_dc10d8d6 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_actor_profile_id_cebde98f field: google_workspace.actor.profile.id copy_from: source.user.id ignore_failure: true - convert: + tag: convert_json_ipAddress_to_source_ip_6cbf48a8 field: json.ipAddress target_field: source.ip type: ip ignore_missing: true on_failure: - append: + tag: append_error_message_79a0c21d field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_ip_address_a932eef3 field: google_workspace.ip_address copy_from: source.ip ignore_failure: true - rename: + tag: rename_json_kind_to_google_workspace_kind_8c6d9f29 field: json.kind target_field: google_workspace.kind ignore_missing: true - rename: + tag: rename_json_etag_to_google_workspace_etag_4bd4a803 field: json.etag target_field: google_workspace.etag ignore_missing: true - convert: + tag: convert_json_id_customerId_to_organization_id_ffd90a33 field: json.id.customerId target_field: organization.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_5aacea86 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_customer_id_2943cbd3 field: google_workspace.id.customer.id copy_from: organization.id ignore_failure: true - rename: + tag: rename_json_actor_callerType_to_google_workspace_actor_type_308eeffc field: json.actor.callerType target_field: google_workspace.actor.type ignore_missing: true - rename: + tag: rename_json_actor_key_to_google_workspace_actor_key_3f392d49 field: json.actor.key target_field: google_workspace.actor.key ignore_missing: true - rename: + tag: rename_json_ownerDomain_to_google_workspace_organization_domain_6580cd7d field: json.ownerDomain target_field: google_workspace.organization.domain ignore_missing: true - rename: + tag: rename_json_events_type_to_google_workspace_event_type_e73f1e18 field: json.events.type target_field: google_workspace.event.type ignore_missing: true - set: + tag: set_user_id_1a22b36e field: user.id copy_from: source.user.id if: ctx.source?.user?.id != null - script: + tag: script_a3a2cdbf lang: painless description: Extract User name and User domain from Email. if: ctx.source?.user?.email != null && ctx.source.user.email.contains("@") @@ -183,30 +218,35 @@ processors: ctx.user.domain = splitmail[1]; ctx.source.user.domain = splitmail[1]; - append: + tag: append_related_hosts_6782fd8a field: related.hosts value: '{{{user.domain}}}' if: ctx.user?.domain != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_hosts_49ff5a59 field: related.hosts value: '{{{google_workspace.organization.domain}}}' if: ctx.google_workspace?.organization?.domain != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_ip_30d15214 field: related.ip value: '{{{source.ip}}}' if: ctx.source?.ip != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_fb2b187b field: related.user value: '{{{source.user.name}}}' if: ctx.source?.user?.name != null allow_duplicates: false ignore_failure: true - script: + tag: script_1a7fee67 lang: painless description: Map Event parameters into key value pair. if: ctx.json?.events?.parameters != null && ctx.json.events.parameters instanceof List @@ -304,62 +344,78 @@ processors: } } - convert: + tag: convert_google_workspace_rules_actor_ip_address_875456f3 field: google_workspace.rules.actor_ip_address type: ip ignore_missing: true on_failure: - remove: + tag: remove_google_workspace_rules_actor_ip_address_4cb7b9a7 field: google_workspace.rules.actor_ip_address - append: + tag: append_error_message_a6eae43c field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_google_workspace_rules_device_id_to_google_workspace_rules_device_id_f6a3bc69 field: google_workspace.rules.device_id target_field: google_workspace.rules.device.id ignore_missing: true - rename: + tag: rename_google_workspace_rules_device_type_to_google_workspace_rules_device_type_212eb24d field: google_workspace.rules.device_type target_field: google_workspace.rules.device.type ignore_missing: true - convert: + tag: convert_google_workspace_rules_has_alert_08a3508c field: google_workspace.rules.has_alert type: boolean ignore_missing: true on_failure: - remove: + tag: remove_google_workspace_rules_has_alert_a1b94c2d field: google_workspace.rules.has_alert - append: + tag: append_error_message_9cb53865 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_google_workspace_rules_matched_detectors_to_google_workspace_rules_matched_detectors_ca7f0d8b field: google_workspace.rules.matched_detectors target_field: google_workspace.rules.matched.detectors ignore_missing: true - rename: + tag: rename_google_workspace_rules_matched_threshold_to_google_workspace_rules_matched_threshold_d3fdbf6f field: google_workspace.rules.matched_threshold target_field: google_workspace.rules.matched.threshold ignore_missing: true - rename: + tag: rename_google_workspace_rules_matched_trigger_to_google_workspace_rules_matched_trigger_cb050abb field: google_workspace.rules.matched_trigger target_field: google_workspace.rules.matched.trigger ignore_missing: true - rename: + tag: rename_google_workspace_rules_matched_templates_to_google_workspace_rules_matched_templates_b121ff6f field: google_workspace.rules.matched_templates target_field: google_workspace.rules.matched.templates ignore_missing: true - rename: + tag: rename_google_workspace_rules_resource_id_to_google_workspace_rules_resource_id_3a8cddd5 field: google_workspace.rules.resource_id target_field: google_workspace.rules.resource.id ignore_missing: true - rename: + tag: rename_google_workspace_rules_resource_owner_email_to_google_workspace_rules_resource_owner_email_baf400a3 field: google_workspace.rules.resource_owner_email target_field: google_workspace.rules.resource.owner_email ignore_missing: true - rename: + tag: rename_google_workspace_rules_resource_recipients_to_google_workspace_rules_resource_recipients_464383f5 field: google_workspace.rules.resource_recipients target_field: google_workspace.rules.resource.recipients ignore_missing: true - script: + tag: script_be0ba7ad lang: painless description: Add recipients domains to related.hosts. if: ctx.google_workspace?.rules?.resource?.recipients != null @@ -392,74 +448,93 @@ processors: ctx.related.hosts = related_domains; ignore_failure: true - convert: + tag: convert_google_workspace_rules_resource_recipients_omitted_count_to_google_workspace_rules_resource_recipients_omitted_count_3e0b46c9 field: google_workspace.rules.resource_recipients_omitted_count target_field: google_workspace.rules.resource.recipients_omitted_count type: long ignore_missing: true on_failure: - remove: + tag: remove_google_workspace_rules_resource_recipients_omitted_count_09f06c13 field: google_workspace.rules.resource_recipients_omitted_count - append: + tag: append_error_message_b2bafa8a field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_google_workspace_rules_resource_title_to_google_workspace_rules_resource_title_99cd434f field: google_workspace.rules.resource_title target_field: google_workspace.rules.resource.title ignore_missing: true - rename: + tag: rename_google_workspace_rules_resource_type_to_google_workspace_rules_resource_type_1f844b7d field: google_workspace.rules.resource_type target_field: google_workspace.rules.resource.type ignore_missing: true - rename: + tag: rename_google_workspace_rules_rule_name_to_rule_name_2adfc50b field: google_workspace.rules.rule_name target_field: rule.name ignore_missing: true - set: + tag: set_google_workspace_rules_name_38060abc field: google_workspace.rules.name copy_from: rule.name ignore_failure: true - rename: + tag: rename_google_workspace_rules_rule_resource_name_to_google_workspace_rules_resource_name_588d1e04 field: google_workspace.rules.rule_resource_name target_field: google_workspace.rules.resource.name ignore_missing: true - rename: + tag: rename_google_workspace_rules_rule_type_to_google_workspace_rules_type_d0e88c3b field: google_workspace.rules.rule_type target_field: google_workspace.rules.type ignore_missing: true - rename: + tag: rename_google_workspace_rules_space_id_to_google_workspace_rules_space_id_60fb5a9f field: google_workspace.rules.space_id target_field: google_workspace.rules.space.id ignore_missing: true - rename: + tag: rename_google_workspace_rules_space_type_to_google_workspace_rules_space_type_e88f8e4d field: google_workspace.rules.space_type target_field: google_workspace.rules.space.type ignore_missing: true - convert: + tag: convert_google_workspace_rules_has_content_match_69ed45b8 field: google_workspace.rules.has_content_match type: boolean ignore_missing: true on_failure: - remove: + tag: remove_google_workspace_rules_has_content_match_1060c582 field: google_workspace.rules.has_content_match - append: + tag: append_error_message_62952c61 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_google_workspace_rules_rule_id_to_rule_id_a7bbe100 field: google_workspace.rules.rule_id target_field: rule.id type: string ignore_missing: true on_failure: - remove: + tag: remove_google_workspace_rules_rule_id_d31c9f20 field: google_workspace.rules.rule_id - append: + tag: append_error_message_6348f091 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_rules_id_23d576c0 field: google_workspace.rules.id copy_from: rule.id ignore_failure: true - date: + tag: date_google_workspace_rules_rule_update_time_usec_to_google_workspace_rules_update_time_usec_4abe924e field: google_workspace.rules.rule_update_time_usec target_field: google_workspace.rules.update_time_usec if: ctx.google_workspace?.rules?.rule_update_time_usec != null && ctx.google_workspace.rules.rule_update_time_usec != '' @@ -472,13 +547,16 @@ processors: - yyyy/MM/dd HH:mm:ss z on_failure: - append: + tag: append_error_message_88cd067f field: error.message value: '{{{_ingest.on_failure_message}}}' - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -487,14 +565,17 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - append: + tag: append_related_ip_6b6c6c4b field: related.ip value: '{{{google_workspace.rules.actor_ip_address}}}' if: ctx.google_workspace?.rules?.actor_ip_address != null @@ -530,6 +611,7 @@ processors: value: alert if: ctx.google_workspace?.rules?.has_alert == true - remove: + tag: remove_0a8a98bf field: - json - google_workspace.rules.resource_recipients_omitted_count @@ -537,6 +619,7 @@ processors: - google_workspace.rules.rule_update_time_usec ignore_missing: true - remove: + tag: remove_1b28c498 if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - google_workspace.ip_address @@ -552,6 +635,7 @@ processors: ignore_failure: true ignore_missing: true - script: + tag: script_e2ccbdd7 description: Drops null/empty values recursively. lang: painless source: | diff --git a/packages/google_workspace/data_stream/saml/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/saml/elasticsearch/ingest_pipeline/default.yml index 79740e22cca..944c317cc2b 100644 --- a/packages/google_workspace/data_stream/saml/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/saml/elasticsearch/ingest_pipeline/default.yml @@ -13,31 +13,39 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields.s - set: + tag: set_ecs_version_135371bc field: ecs.version value: '8.16.0' - set: + tag: set_event_kind_de80643c field: event.kind value: event - append: + tag: append_event_type_fffb7a5a field: event.type value: start - append: + tag: append_event_category_109b4d72 field: event.category value: authentication - append: + tag: append_event_category_d21db9a2 field: event.category value: session - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - drop: @@ -45,6 +53,7 @@ processors: description: Discard events that are missing the target during the split operation and are subsequently returned as the root object. tag: drop_empty_events - date: + tag: date_json_id_time_dc34ff3c field: json.id.time if: ctx.json?.id?.time != null && ctx.json.id.time != '' timezone: UTC @@ -56,9 +65,11 @@ processors: - yyyy/MM/dd HH:mm:ss z on_failure: - append: + tag: append_error_message_a48f8d5f field: error.message value: "{{{_ingest.on_failure_message}}}" - fingerprint: + tag: fingerprint_567b61c4 description: Hashes the ID object and uses it as the document id to avoid duplicate events. fields: - json.id @@ -67,41 +78,50 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_events_name_to_event_action_75ca640e field: json.events.name target_field: event.action ignore_missing: true - rename: + tag: rename_json_id_applicationName_to_event_provider_fffca35d field: json.id.applicationName target_field: event.provider ignore_missing: true - convert: + tag: convert_json_id_uniqueQualifier_to_event_id_0664525d field: json.id.uniqueQualifier target_field: event.id type: string ignore_missing: true - rename: + tag: rename_json_actor_email_to_source_user_email_4ee5e9ab field: json.actor.email target_field: source.user.email ignore_missing: true - set: + tag: set_user_email_f8429233 field: user.email copy_from: source.user.email ignore_failure: true - convert: + tag: convert_json_actor_profileId_to_source_user_id_edfcffb4 field: json.actor.profileId target_field: source.user.id type: string ignore_missing: true - set: + tag: set_user_id_64f70b73 field: user.id copy_from: source.user.id if: ctx?.source?.user?.id != null - convert: + tag: convert_json_ipAddress_to_source_ip_aabdaf8f field: json.ipAddress target_field: source.ip type: ip ignore_missing: true - rename: + tag: rename_json_kind_to_google_workspace_kind_8c6d9f29 field: json.kind target_field: google_workspace.kind ignore_missing: true @@ -113,25 +133,31 @@ processors: tag: convert_customer_id_to_string on_failure: - append: + tag: append_error_message_044cfe8e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_actor_callerType_to_google_workspace_actor_type_308eeffc field: json.actor.callerType target_field: google_workspace.actor.type ignore_missing: true - rename: + tag: rename_json_actor_key_to_google_workspace_actor_key_3f392d49 field: json.actor.key target_field: google_workspace.actor.key ignore_missing: true - rename: + tag: rename_json_ownerDomain_to_google_workspace_organization_domain_6580cd7d field: json.ownerDomain target_field: google_workspace.organization.domain ignore_missing: true - rename: + tag: rename_json_events_type_to_google_workspace_event_type_e73f1e18 field: json.events.type target_field: google_workspace.event.type ignore_missing: true - script: + tag: script_8d756e1a lang: painless if: 'ctx?.source?.user?.email != null && ctx?.source?.user?.email.contains("@")' source: > @@ -147,24 +173,29 @@ processors: ctx.user.domain = splitmail[1]; ctx.source.user.domain = splitmail[1]; - append: + tag: append_related_ip_549fa858 field: related.ip value: "{{{source.ip}}}" if: ctx?.source?.ip != null allow_duplicates: false - append: + tag: append_related_user_8302ba1f field: related.user value: "{{{source.user.name}}}" if: ctx?.source?.user?.name != null allow_duplicates: false - set: + tag: set_event_outcome_2f7e52b2 field: event.outcome value: success if: 'ctx?.event?.action == "login_success"' - set: + tag: set_event_outcome_c64393de field: event.outcome value: failure if: 'ctx?.event?.action == "login_failure"' - script: + tag: script_6b833f66 lang: painless if: 'ctx?.json?.events?.parameters != null && ctx?.json?.events?.parameters instanceof List' source: > @@ -186,10 +217,12 @@ processors: } } - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -198,14 +231,17 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - remove: + tag: remove_json_29cdffdc field: json ignore_missing: true on_failure: diff --git a/packages/google_workspace/data_stream/token/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/token/elasticsearch/ingest_pipeline/default.yml index 1e7c4f1cfaa..50a54a79011 100644 --- a/packages/google_workspace/data_stream/token/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/token/elasticsearch/ingest_pipeline/default.yml @@ -13,23 +13,28 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields.s - set: + tag: set_ecs_version_135371bc field: ecs.version value: '8.16.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_90c6475f field: event.original target_field: json on_failure: - append: + tag: append_error_message_45b808be field: error.message value: '{{{_ingest.on_failure_message}}}' - drop: @@ -37,12 +42,15 @@ processors: description: Discard events that are missing the target during the split operation and are subsequently returned as the root object. tag: drop_empty_events - set: + tag: set_event_kind_3a8924a2 field: event.kind value: [event] - set: + tag: set_event_category_ee497bfd field: event.category value: [iam] - date: + tag: date_json_id_time_9350ecd5 field: json.id.time if: ctx.json?.id?.time != null && ctx.json.id.time != '' formats: @@ -54,13 +62,16 @@ processors: - yyyy/MM/dd HH:mm:ss z on_failure: - append: + tag: append_error_message_18a82a92 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_time_01ac69ca field: google_workspace.id.time copy_from: '@timestamp' ignore_empty_value: true - fingerprint: + tag: fingerprint_52623c07 description: Hashes the ID object and uses it as the document id to avoid duplicate events. fields: - json.id @@ -68,84 +79,104 @@ processors: target_field: _id ignore_missing: true - rename: + tag: rename_json_events_name_to_event_action_75ca640e field: json.events.name target_field: event.action ignore_missing: true - set: + tag: set_google_workspace_event_name_69e4d7fd field: google_workspace.event.name copy_from: event.action ignore_empty_value: true - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] - append: + tag: append_event_type_0140dff9 field: event.type value: user if: '["authorize", "revoke"].contains(ctx.event?.action?.toLowerCase())' - append: + tag: append_event_type_66c7d15b field: event.type value: deletion if: '["revoke"].contains(ctx.event?.action?.toLowerCase())' - rename: + tag: rename_json_id_applicationName_to_event_provider_fffca35d field: json.id.applicationName target_field: event.provider ignore_missing: true - set: + tag: set_google_workspace_id_application_name_8030da4a field: google_workspace.id.application_name copy_from: event.provider ignore_empty_value: true - convert: + tag: convert_json_id_uniqueQualifier_to_event_id_0239362e field: json.id.uniqueQualifier target_field: event.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_ab6a6ac3 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_unique_qualifier_8c91fe06 field: google_workspace.id.unique_qualifier copy_from: event.id ignore_empty_value: true - rename: + tag: rename_json_actor_email_to_source_user_email_4ee5e9ab field: json.actor.email target_field: source.user.email ignore_missing: true - set: + tag: set_google_workspace_actor_email_9cd77425 field: google_workspace.actor.email copy_from: source.user.email ignore_empty_value: true - set: + tag: set_user_email_371886fa field: user.email copy_from: source.user.email ignore_empty_value: true - convert: + tag: convert_json_actor_profileId_to_source_user_id_e10534f1 field: json.actor.profileId target_field: source.user.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_dc10d8d6 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_actor_profile_id_6b7a4346 field: google_workspace.actor.profile.id copy_from: source.user.id ignore_empty_value: true - convert: + tag: convert_json_ipAddress_to_source_ip_6cbf48a8 field: json.ipAddress target_field: source.ip type: ip ignore_missing: true on_failure: - append: + tag: append_error_message_79a0c21d field: error.message value: '{{{_ingest.on_failure_message}}}' - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -154,59 +185,73 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - set: + tag: set_google_workspace_ip_address_b61dbbba field: google_workspace.ip_address copy_from: source.ip ignore_empty_value: true - rename: + tag: rename_json_kind_to_google_workspace_kind_8c6d9f29 field: json.kind target_field: google_workspace.kind ignore_missing: true - rename: + tag: rename_json_etag_to_google_workspace_etag_4bd4a803 field: json.etag target_field: google_workspace.etag ignore_missing: true - convert: + tag: convert_json_id_customerId_to_organization_id_ffd90a33 field: json.id.customerId target_field: organization.id type: string ignore_missing: true on_failure: - append: + tag: append_error_message_5aacea86 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_google_workspace_id_customer_id_46a3415a field: google_workspace.id.customer.id copy_from: organization.id ignore_empty_value: true - rename: + tag: rename_json_actor_callerType_to_google_workspace_actor_type_308eeffc field: json.actor.callerType target_field: google_workspace.actor.type ignore_missing: true - rename: + tag: rename_json_actor_key_to_google_workspace_actor_key_3f392d49 field: json.actor.key target_field: google_workspace.actor.key ignore_missing: true - rename: + tag: rename_json_ownerDomain_to_google_workspace_organization_domain_6580cd7d field: json.ownerDomain target_field: google_workspace.organization.domain ignore_missing: true - rename: + tag: rename_json_events_type_to_google_workspace_event_type_e73f1e18 field: json.events.type target_field: google_workspace.event.type ignore_missing: true - set: + tag: set_user_id_f89889ac field: user.id copy_from: source.user.id ignore_empty_value: true - script: + tag: script_a3a2cdbf lang: painless description: Extract User name and User domain from Email. if: ctx.source?.user?.email != null && ctx.source.user.email.contains("@") @@ -223,36 +268,43 @@ processors: ctx.user.domain = splitmail[1]; ctx.source.user.domain = splitmail[1]; - append: + tag: append_related_hosts_23e97a1f field: related.hosts value: '{{{user.domain}}}' if: ctx.user?.domain != null allow_duplicates: false - append: + tag: append_related_hosts_ea7c8eda field: related.hosts value: '{{{google_workspace.organization.domain}}}' if: ctx.google_workspace?.organization?.domain != null allow_duplicates: false - append: + tag: append_related_ip_8121c591 field: related.ip value: '{{{source.ip}}}' if: ctx.source?.ip != null allow_duplicates: false - append: + tag: append_related_user_bcb5a560 field: related.user value: '{{{source.user.id}}}' if: ctx.source?.user?.id != null allow_duplicates: false - append: + tag: append_related_user_f745a124 field: related.user value: '{{{source.user.name}}}' if: ctx.source?.user?.name != null allow_duplicates: false - append: + tag: append_related_user_8c49e24b field: related.user value: '{{{source.user.email}}}' if: ctx.source?.user?.name != null allow_duplicates: false - script: + tag: script_267008ad lang: painless description: Map Event parameters into key value pair. if: ctx.json?.events?.parameters != null && ctx.json.events.parameters instanceof List @@ -324,34 +376,42 @@ processors: ctx.json.events.parameters = parameters; } - rename: + tag: rename_google_workspace_token_client_id_to_google_workspace_token_client_id_ffdaad29 field: google_workspace.token.client_id target_field: google_workspace.token.client.id ignore_missing: true - rename: + tag: rename_google_workspace_token_client_type_to_google_workspace_token_client_type_dcef4099 field: google_workspace.token.client_type target_field: google_workspace.token.client.type ignore_missing: true - convert: + tag: convert_google_workspace_token_num_response_bytes_2b609322 field: google_workspace.token.num_response_bytes type: long ignore_missing: true on_failure: - append: + tag: append_error_message_beff5177 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_google_workspace_token_scope_to_google_workspace_token_scope_value_b334a2fd field: google_workspace.token.scope target_field: google_workspace.token.scope.value ignore_missing: true - rename: + tag: rename_google_workspace_token_scope_data_to_google_workspace_token_scope_data_15f11cf5 field: google_workspace.token.scope_data target_field: google_workspace.token.scope.data ignore_missing: true - remove: + tag: remove_e5a80934 field: - json ignore_missing: true - remove: + tag: remove_2a1ed52f if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - google_workspace.ip_address @@ -364,6 +424,7 @@ processors: - google_workspace.id.time ignore_missing: true - script: + tag: script_e2ccbdd7 description: Drops null/empty values recursively. lang: painless source: | @@ -381,6 +442,7 @@ processors: } dropEmptyFields(ctx); - set: + tag: set_event_kind_92954dfa field: event.kind value: pipeline_error if: ctx.error?.message != null diff --git a/packages/google_workspace/data_stream/user_accounts/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/user_accounts/elasticsearch/ingest_pipeline/default.yml index c477d7a85b7..110f2d48c96 100644 --- a/packages/google_workspace/data_stream/user_accounts/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/user_accounts/elasticsearch/ingest_pipeline/default.yml @@ -13,31 +13,39 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - set: + tag: set_ecs_version_135371bc field: ecs.version value: '8.16.0' - set: + tag: set_event_kind_de80643c field: event.kind value: event - append: + tag: append_event_type_0602e5be field: event.type value: change - append: + tag: append_event_type_7ca34f6b field: event.type value: user - append: + tag: append_event_category_576de059 field: event.category value: iam - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - drop: @@ -45,6 +53,7 @@ processors: description: Discard events that are missing the target during the split operation and are subsequently returned as the root object. tag: drop_empty_events - date: + tag: date_json_id_time_dc34ff3c field: json.id.time if: ctx.json?.id?.time != null && ctx.json.id.time != '' timezone: UTC @@ -56,9 +65,11 @@ processors: - yyyy/MM/dd HH:mm:ss z on_failure: - append: + tag: append_error_message_a48f8d5f field: error.message value: "{{{_ingest.on_failure_message}}}" - fingerprint: + tag: fingerprint_567b61c4 description: Hashes the ID object and uses it as the document id to avoid duplicate events. fields: - json.id @@ -67,41 +78,50 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_events_name_to_event_action_75ca640e field: json.events.name target_field: event.action ignore_missing: true - rename: + tag: rename_json_id_applicationName_to_event_provider_fffca35d field: json.id.applicationName target_field: event.provider ignore_missing: true - convert: + tag: convert_json_id_uniqueQualifier_to_event_id_0664525d field: json.id.uniqueQualifier target_field: event.id type: string ignore_missing: true - rename: + tag: rename_json_actor_email_to_source_user_email_4ee5e9ab field: json.actor.email target_field: source.user.email ignore_missing: true - set: + tag: set_user_email_f8429233 field: user.email copy_from: source.user.email ignore_failure: true - convert: + tag: convert_json_actor_profileId_to_source_user_id_edfcffb4 field: json.actor.profileId target_field: source.user.id type: string ignore_missing: true - set: + tag: set_user_id_64f70b73 field: user.id copy_from: source.user.id if: ctx?.source?.user?.id != null - convert: + tag: convert_json_ipAddress_to_source_ip_aabdaf8f field: json.ipAddress target_field: source.ip type: ip ignore_missing: true - rename: + tag: rename_json_kind_to_google_workspace_kind_8c6d9f29 field: json.kind target_field: google_workspace.kind ignore_missing: true @@ -113,21 +133,26 @@ processors: tag: convert_customer_id_to_string on_failure: - append: + tag: append_error_message_044cfe8e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_actor_callerType_to_google_workspace_actor_type_308eeffc field: json.actor.callerType target_field: google_workspace.actor.type ignore_missing: true - rename: + tag: rename_json_actor_key_to_google_workspace_actor_key_3f392d49 field: json.actor.key target_field: google_workspace.actor.key ignore_missing: true - rename: + tag: rename_json_ownerDomain_to_google_workspace_organization_domain_6580cd7d field: json.ownerDomain target_field: google_workspace.organization.domain ignore_missing: true - rename: + tag: rename_json_events_type_to_google_workspace_event_type_e73f1e18 field: json.events.type target_field: google_workspace.event.type ignore_missing: true @@ -153,9 +178,11 @@ processors: } } - remove: + tag: remove_json_events_parameters_2eab3501 field: json.events.parameters ignore_missing: true - script: + tag: script_8d756e1a lang: painless if: 'ctx?.source?.user?.email != null && ctx?.source?.user?.email.contains("@")' source: > @@ -171,20 +198,24 @@ processors: ctx.user.domain = splitmail[1]; ctx.source.user.domain = splitmail[1]; - append: + tag: append_related_ip_549fa858 field: related.ip value: "{{{source.ip}}}" if: ctx?.source?.ip != null allow_duplicates: false - append: + tag: append_related_user_8302ba1f field: related.user value: "{{{source.user.name}}}" if: ctx?.source?.user?.name != null allow_duplicates: false - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -193,14 +224,17 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - remove: + tag: remove_json_29cdffdc field: json ignore_missing: true on_failure: diff --git a/packages/google_workspace/data_stream/vault/elasticsearch/ingest_pipeline/default.yml b/packages/google_workspace/data_stream/vault/elasticsearch/ingest_pipeline/default.yml index a944e52b56c..cbd4ee03dec 100644 --- a/packages/google_workspace/data_stream/vault/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_workspace/data_stream/vault/elasticsearch/ingest_pipeline/default.yml @@ -48,6 +48,7 @@ processors: if: ctx.event?.original != null on_failure: - append: + tag: append_error_message_04014d20 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -79,9 +80,11 @@ processors: } on_failure: - append: + tag: append_error_message_22f11310 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: + tag: remove_json_events_parameters_2eab3501 field: json.events.parameters ignore_missing: true - rename: @@ -193,6 +196,7 @@ processors: } on_failure: - append: + tag: append_error_message_6b2567f2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -298,6 +302,7 @@ processors: } on_failure: - append: + tag: append_error_message_18547902 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -305,7 +310,8 @@ processors: tag: rename_json_kind target_field: google_workspace.kind ignore_missing: true - - rename: + - rename: + tag: rename_json_ownerDomain_to_google_workspace_organization_domain_6580cd7d field: json.ownerDomain target_field: google_workspace.organization.domain ignore_missing: true @@ -341,6 +347,7 @@ processors: pattern: '%{}@%{source.user.domain}' on_failure: - append: + tag: append_error_message_4450a0e8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -393,6 +400,7 @@ processors: if: ctx.json?.id?.time != null && ctx.json.id.time != '' on_failure: - append: + tag: append_error_message_de0bed15 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -419,6 +427,7 @@ processors: if: ctx.json?.ipAddress != '' on_failure: - append: + tag: append_error_message_4cb1517b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -441,10 +450,12 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true @@ -467,6 +478,7 @@ processors: if: ctx.event?.action != '' on_failure: - append: + tag: append_error_message_32466e0d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -494,6 +506,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2ee52bde field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -517,6 +530,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2cac5b72 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -526,6 +540,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a2b294cd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -535,6 +550,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_584a3241 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -544,6 +560,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8259aa95 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -615,6 +632,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/google_workspace/manifest.yml b/packages/google_workspace/manifest.yml index a8d041f26aa..8d9ddc60b0b 100644 --- a/packages/google_workspace/manifest.yml +++ b/packages/google_workspace/manifest.yml @@ -1,6 +1,6 @@ name: google_workspace title: Google Workspace -version: "3.5.2" +version: "3.5.3" source: license: Elastic-2.0 description: Collect logs from Google Workspace with Elastic Agent. diff --git a/packages/greenhouse/changelog.yml b/packages/greenhouse/changelog.yml index 0a1cb430a6f..f5af6dc3bba 100644 --- a/packages/greenhouse/changelog.yml +++ b/packages/greenhouse/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.1.1" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "0.1.0" changes: - description: Initial release. diff --git a/packages/greenhouse/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/greenhouse/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 64660b5324b..b70c22f496e 100644 --- a/packages/greenhouse/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/greenhouse/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -5,14 +5,17 @@ processors: tag: terminate_agent_error if: ctx.error?.message != null && ctx.message == null && ctx.event?.original == null - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: 8.11.0 - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: ctx.event?.original != null @@ -22,9 +25,11 @@ processors: target_field: json tag: parse_json - fail: + tag: fail_970014bf if: ctx.json == null || !(ctx.json instanceof Map) message: Missing JSON object in input document - fingerprint: + tag: fingerprint_8645335e fields: - json.organization_id - json.request.id @@ -33,6 +38,7 @@ processors: target_field: _id ignore_missing: true - rename: + tag: rename_json_to_greenhouse_audit_41e670b6 field: json target_field: greenhouse.audit ignore_missing: true @@ -48,80 +54,96 @@ processors: tag: parse_timestamp on_failure: - append: + tag: append_error_message_a30c2847 field: error.message value: 'Failed to parse timestamp from greenhouse.audit.event_time: {{{ _ingest.on_failure_message }}}' # Event fields - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_id_8953a766 field: event.id copy_from: greenhouse.audit.request.id ignore_empty_value: true - set: + tag: set_event_action_8eabdebd field: event.action copy_from: greenhouse.audit.event.type ignore_empty_value: true # Event type categorization based on Greenhouse event types - append: + tag: append_event_type_6b5cb0e5 field: event.type value: creation if: ctx.greenhouse?.audit?.event?.type != null && ctx.greenhouse.audit.event.type == 'data_change_create' allow_duplicates: false - append: + tag: append_event_type_6d53c86d field: event.type value: change if: ctx.greenhouse?.audit?.event?.type != null && ctx.greenhouse.audit.event.type == 'data_change_update' allow_duplicates: false - append: + tag: append_event_type_59f63d12 field: event.type value: deletion if: ctx.greenhouse?.audit?.event?.type != null && ctx.greenhouse.audit.event.type == 'data_change_destroy' allow_duplicates: false - append: + tag: append_event_type_72c5552a field: event.type value: access if: ctx.greenhouse?.audit?.event?.type != null && ctx.greenhouse.audit.event.type == 'harvest_access' allow_duplicates: false - append: + tag: append_event_type_d9d89ce6 field: event.type value: info if: ctx.greenhouse?.audit?.event?.type != null && ctx.greenhouse.audit.event.type == 'action' allow_duplicates: false - set: + tag: set_event_type_e4a41200 field: event.type value: [info] if: ctx.event?.type == null # Event category - append: + tag: append_event_category_893e6a4f field: event.category value: configuration if: ctx.greenhouse?.audit?.event?.type != null && ctx.greenhouse.audit.event.type.startsWith('data_change') allow_duplicates: false - append: + tag: append_event_category_38235bd4 field: event.category value: iam if: ctx.greenhouse?.audit?.performer?.type != null && ctx.greenhouse.audit.performer.type == 'user' allow_duplicates: false - set: + tag: set_event_category_87dd0ae8 field: event.category value: [configuration] if: ctx.event?.category == null # User fields from performer - convert: + tag: convert_greenhouse_audit_performer_id_to_user_id_d0c2835d field: greenhouse.audit.performer.id type: string target_field: user.id ignore_missing: true - set: + tag: set_user_email_e65619b3 field: user.email copy_from: greenhouse.audit.performer.meta.username ignore_empty_value: true - set: + tag: set_user_full_name_1702943f field: user.full_name copy_from: greenhouse.audit.performer.meta.name ignore_empty_value: true @@ -135,6 +157,7 @@ processors: tag: convert_ip_address on_failure: - append: + tag: append_error_message_ebffbae2 field: error.message value: 'Failed to convert performer.ip_address to IP: {{{ _ingest.on_failure_message }}}' @@ -156,10 +179,12 @@ processors: ignore_missing: true tag: geoip_source_asn - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true @@ -173,6 +198,7 @@ processors: tag: convert_organization_id on_failure: - append: + tag: append_error_message_6172a02a field: error.message value: 'Failed to convert organization_id to string: {{{ _ingest.on_failure_message }}}' @@ -184,21 +210,25 @@ processors: tag: convert_target_id on_failure: - append: + tag: append_error_message_d3bb095c field: error.message value: 'Failed to convert target_id to string: {{{ _ingest.on_failure_message }}}' # Related fields - append: + tag: append_related_user_7f407fef field: related.user value: '{{{user.id}}}' if: ctx.user?.id != null allow_duplicates: false - append: + tag: append_related_user_34fcf415 field: related.user value: '{{{user.email}}}' if: ctx.user?.email != null allow_duplicates: false - append: + tag: append_related_ip_8121c591 field: related.ip value: '{{{source.ip}}}' if: ctx.source?.ip != null @@ -206,6 +236,7 @@ processors: # Cleanup null/empty values - script: + tag: script_4d6993cb lang: painless description: This script processor iterates over the whole document to remove fields with null values. source: | @@ -233,6 +264,7 @@ processors: # Remove temporary fields - remove: + tag: remove_122ff8dd field: - greenhouse.audit.event_time - greenhouse.audit.performer.id diff --git a/packages/greenhouse/manifest.yml b/packages/greenhouse/manifest.yml index fa5b99bca6f..491178dea7c 100644 --- a/packages/greenhouse/manifest.yml +++ b/packages/greenhouse/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.3.2" name: greenhouse title: "Greenhouse" -version: "0.1.0" +version: "0.1.1" description: Collect audit logs from Greenhouse ATS with Elastic Agent. type: integration categories: diff --git a/packages/ibm_qradar/changelog.yml b/packages/ibm_qradar/changelog.yml index ae3b2c5e1f8..0d5643a5899 100644 --- a/packages/ibm_qradar/changelog.yml +++ b/packages/ibm_qradar/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.1.3" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.1.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/ibm_qradar/data_stream/offense/elasticsearch/ingest_pipeline/default.yml b/packages/ibm_qradar/data_stream/offense/elasticsearch/ingest_pipeline/default.yml index 517afb52fe2..5d092208429 100644 --- a/packages/ibm_qradar/data_stream/offense/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ibm_qradar/data_stream/offense/elasticsearch/ingest_pipeline/default.yml @@ -25,7 +25,6 @@ processors: description: >- Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - # parse the event JSON - rename: field: message @@ -89,8 +88,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ibm_qradar_offense_category_count_ff5d88ce field: ibm_qradar.offense.category_count - append: + tag: append_error_message_f9dc9732 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -102,6 +103,7 @@ processors: ctx.event.duration = (ctx.ibm_qradar.offense.close_time - ctx.ibm_qradar.offense.start_time) * 1000000; on_failure: - append: + tag: append_error_message_fb23327c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -113,8 +115,10 @@ processors: if: ctx.ibm_qradar?.offense?.start_time != null on_failure: - remove: + tag: remove_ibm_qradar_offense_start_time_2af8d8c0 field: ibm_qradar.offense.start_time - append: + tag: append_error_message_84b9072c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -131,8 +135,10 @@ processors: if: ctx.ibm_qradar?.offense?.close_time != null on_failure: - remove: + tag: remove_ibm_qradar_offense_close_time_59f94654 field: ibm_qradar.offense.close_time - append: + tag: append_error_message_befdd7c4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -152,8 +158,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ibm_qradar_offense_credibility_dca155b2 field: ibm_qradar.offense.credibility - append: + tag: append_error_message_ce72caca field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -168,8 +176,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ibm_qradar_offense_device_count_460e807a field: ibm_qradar.offense.device_count - append: + tag: append_error_message_286a5480 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -184,8 +194,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ibm_qradar_offense_event_count_935280c6 field: ibm_qradar.offense.event_count - append: + tag: append_error_message_47344c24 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -197,8 +209,10 @@ processors: if: ctx.ibm_qradar?.offense?.first_persisted_time != null on_failure: - remove: + tag: remove_ibm_qradar_offense_first_persisted_time_76fb3804 field: ibm_qradar.offense.first_persisted_time - append: + tag: append_error_message_d7980e54 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -213,8 +227,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ibm_qradar_offense_flow_count_39f8ed92 field: ibm_qradar.offense.flow_count - append: + tag: append_error_message_25affdca field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -224,8 +240,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ibm_qradar_offense_follow_up_025fda9a field: ibm_qradar.offense.follow_up - append: + tag: append_error_message_d255b8d9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -245,8 +263,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ibm_qradar_offense_inactive_944f179e field: ibm_qradar.offense.inactive - append: + tag: append_error_message_ff49a52f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -258,8 +278,10 @@ processors: if: ctx.ibm_qradar?.offense?.last_persisted_time != null on_failure: - remove: + tag: remove_ibm_qradar_offense_last_persisted_time_2444d6a4 field: ibm_qradar.offense.last_persisted_time - append: + tag: append_error_message_6de63a44 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -276,8 +298,10 @@ processors: if: ctx.ibm_qradar?.offense?.last_updated_time != null on_failure: - remove: + tag: remove_ibm_qradar_offense_last_updated_time_82222f20 field: ibm_qradar.offense.last_updated_time - append: + tag: append_error_message_ae0b8baa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -292,8 +316,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ibm_qradar_offense_local_destination_count_634b013e field: ibm_qradar.offense.local_destination_count - append: + tag: append_error_message_333ae648 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -308,7 +334,7 @@ processors: ignore_missing: true - foreach: field: ibm_qradar.offense.log_sources - tag: foreach_log_source + tag: foreach_ibm_qradar_offense_log_sources_83a3ec01 if: ctx.ibm_qradar?.offense?.log_sources instanceof List processor: convert: @@ -323,8 +349,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ibm_qradar_offense_magnitude_41968e5a field: ibm_qradar.offense.magnitude - append: + tag: append_error_message_c45a27c2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -339,8 +367,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ibm_qradar_offense_policy_category_count_abb89c5e field: ibm_qradar.offense.policy_category_count - append: + tag: append_error_message_a528af99 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -350,8 +380,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ibm_qradar_offense_protected_057471b6 field: ibm_qradar.offense.protected - append: + tag: append_error_message_5812160e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -361,8 +393,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ibm_qradar_offense_relevance_0a0ebb0e field: ibm_qradar.offense.relevance - append: + tag: append_error_message_13b38785 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -372,8 +406,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ibm_qradar_offense_remote_destination_count_05dfcab2 field: ibm_qradar.offense.remote_destination_count - append: + tag: append_error_message_ce30c167 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -494,7 +530,7 @@ processors: ignore_missing: true - foreach: field: ibm_qradar.offense.rules - tag: foreach_rules_id + tag: foreach_ibm_qradar_offense_rules_27a25c89 if: ctx.ibm_qradar?.offense?.rules instanceof List processor: convert: @@ -504,7 +540,7 @@ processors: ignore_missing: true - foreach: field: ibm_qradar.offense.rules - tag: foreach_rules_id + tag: foreach_ibm_qradar_offense_rules_755f65c0 if: ctx.ibm_qradar?.offense?.rules instanceof List processor: append: @@ -549,8 +585,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ibm_qradar_offense_security_category_count_f2cdcbb6 field: ibm_qradar.offense.security_category_count - append: + tag: append_error_message_c9e0b5d9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -560,8 +598,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ibm_qradar_offense_severity_601fe3be field: ibm_qradar.offense.severity - append: + tag: append_error_message_19b022f5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -582,6 +622,7 @@ processors: } on_failure: - append: + tag: append_error_message_10f86543 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -596,8 +637,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ibm_qradar_offense_source_count_93a351ae field: ibm_qradar.offense.source_count - append: + tag: append_error_message_772b60a7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -607,8 +650,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ibm_qradar_offense_username_count_b3ec796a field: ibm_qradar.offense.username_count - append: + tag: append_error_message_36f605d6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' @@ -673,6 +718,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/ibm_qradar/manifest.yml b/packages/ibm_qradar/manifest.yml index e047695e13f..9612bb81979 100644 --- a/packages/ibm_qradar/manifest.yml +++ b/packages/ibm_qradar/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: ibm_qradar title: IBM QRadar -version: 1.1.2 +version: 1.1.3 description: Collect logs from IBM QRadar with Elastic Agent. type: integration categories: diff --git a/packages/imperva_cloud_waf/changelog.yml b/packages/imperva_cloud_waf/changelog.yml index a8b902011d1..1334d9c460c 100644 --- a/packages/imperva_cloud_waf/changelog.yml +++ b/packages/imperva_cloud_waf/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.15.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.15.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/imperva_cloud_waf/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/imperva_cloud_waf/data_stream/event/elasticsearch/ingest_pipeline/default.yml index 2e0b79f1a3b..f2edb4a8979 100644 --- a/packages/imperva_cloud_waf/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/imperva_cloud_waf/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -72,9 +72,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_event_action_c8457770 field: event.action ignore_missing: true - append: + tag: append_error_message_caf4fe17 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - json: @@ -84,6 +86,7 @@ processors: if: ctx.cef?.extensions?.additionalReqHeaders != null && ctx.cef.extensions.additionalReqHeaders != '' on_failure: - append: + tag: append_error_message_fcc25d1c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - json: @@ -93,6 +96,7 @@ processors: if: ctx.cef?.extensions?.additionalResHeaders != null && ctx.cef.extensions.additionalResHeaders != '' on_failure: - append: + tag: append_error_message_a6fb4542 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -112,6 +116,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ec359c55 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -136,6 +141,7 @@ processors: if: ctx.cef?.extensions?.cs10 != null && ctx.cef.extensions.cs10 != '' on_failure: - append: + tag: append_error_message_4f043bef field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -150,6 +156,7 @@ processors: if: ctx.cef?.extensions?.cs11 != null && ctx.cef.extensions.cs11 != '' on_failure: - append: + tag: append_error_message_0596941a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -166,6 +173,7 @@ processors: if: ctx.cef?.extensions?.cs7 != null && ctx.cef.extensions.cs7 != '' on_failure: - append: + tag: append_error_message_d6e5e3e3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -182,6 +190,7 @@ processors: if: ctx.cef?.extensions?.cs8 != null && ctx.cef.extensions.cs8 != '' on_failure: - append: + tag: append_error_message_cb6f5056 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -252,6 +261,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_97281043 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -267,6 +277,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_1094f286 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -323,6 +334,7 @@ processors: if: ctx.cef?.extensions?.endTime != null && ctx.cef.extensions.endTime != '' on_failure: - append: + tag: append_error_message_28f58597 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -363,9 +375,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_url_original_566accf5 field: url.original ignore_missing: true - append: + tag: append_error_message_dba29050 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -380,6 +394,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ea7094ee field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - user_agent: @@ -389,6 +404,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d82c4066 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -425,6 +441,7 @@ processors: if: ctx.cef?.extensions?.sip != null && ctx.cef.extensions.sip != '' on_failure: - append: + tag: append_error_message_9663fd88 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -457,6 +474,7 @@ processors: if: ctx.cef?.extensions?.sourceAddress != null && ctx.cef.extensions.sourceAddress != '' on_failure: - append: + tag: append_error_message_58bf35cd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -473,6 +491,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5f1fc79a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: @@ -493,6 +512,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e9500be5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -525,6 +545,7 @@ processors: if: ctx.cef?.extensions?.startTime != null && ctx.cef.extensions.startTime != '' on_failure: - append: + tag: append_error_message_0b45e276 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -545,6 +566,7 @@ processors: - ^(TLSv%{GREEDYDATA:tls.version} %{GREEDYDATA:tls.cipher})$ on_failure: - append: + tag: append_error_message_abf661b5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - split: @@ -555,6 +577,7 @@ processors: if: ctx.cef?.extensions?.xff instanceof String on_failure: - append: + tag: append_error_message_34d55f34 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -608,6 +631,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f68755b0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -672,11 +696,13 @@ processors: drop(ctx); description: Drops null/empty values recursively. - append: + tag: append_event_kind_d2f25e35 field: event.kind value: pipeline_error allow_duplicates: false if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/imperva_cloud_waf/manifest.yml b/packages/imperva_cloud_waf/manifest.yml index 56f3bd1ae8e..f08d2189110 100644 --- a/packages/imperva_cloud_waf/manifest.yml +++ b/packages/imperva_cloud_waf/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: imperva_cloud_waf title: Imperva Cloud WAF -version: "1.15.1" +version: "1.15.2" description: Collect logs from Imperva Cloud WAF with Elastic Agent. type: integration categories: diff --git a/packages/infoblox_bloxone_ddi/changelog.yml b/packages/infoblox_bloxone_ddi/changelog.yml index 88c7e57c175..f2e4e8bc791 100644 --- a/packages/infoblox_bloxone_ddi/changelog.yml +++ b/packages/infoblox_bloxone_ddi/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.23.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.23.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/elasticsearch/ingest_pipeline/default.yml b/packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/elasticsearch/ingest_pipeline/default.yml index fc68ae47346..82f41085aa8 100644 --- a/packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/elasticsearch/ingest_pipeline/default.yml +++ b/packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/elasticsearch/ingest_pipeline/default.yml @@ -2,18 +2,23 @@ description: Pipeline for parsing DHCP lease logs. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - set: + tag: set_event_type_42379782 field: event.type value: [protocol] - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -25,11 +30,14 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - drop: + tag: drop_88d6232c if: ctx.json?.results instanceof List && ctx.json.results.length == 0 - fingerprint: + tag: fingerprint_5163c038 fields: - json.starts - json.last_updated @@ -37,6 +45,7 @@ processors: target_field: _id ignore_missing: true - convert: + tag: convert_json_address_to_infoblox_bloxone_ddi_dhcp_lease_address_7777cb8b field: json.address target_field: infoblox_bloxone_ddi.dhcp_lease.address if: ctx.json?.address != '' @@ -44,22 +53,27 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_88799948 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_related_ip_d8950161 field: related.ip value: '{{{infoblox_bloxone_ddi.dhcp_lease.address}}}' allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_client_id_to_infoblox_bloxone_ddi_dhcp_lease_client_id_5bda9385 field: json.client_id target_field: infoblox_bloxone_ddi.dhcp_lease.client_id ignore_missing: true - set: + tag: set_client_user_id_d6b5d0f9 field: client.user.id copy_from: infoblox_bloxone_ddi.dhcp_lease.client_id ignore_failure: true - date: + tag: date_json_ends_to_infoblox_bloxone_ddi_dhcp_lease_ends_eb4503fe field: json.ends target_field: infoblox_bloxone_ddi.dhcp_lease.ends if: ctx.json?.ends != null && ctx.json.ends != '' @@ -67,65 +81,80 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_137b2977 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_event_end_63e76756 field: event.end copy_from: infoblox_bloxone_ddi.dhcp_lease.ends ignore_failure: true - rename: + tag: rename_json_fingerprint_to_infoblox_bloxone_ddi_dhcp_lease_fingerprint_value_846eae7e field: json.fingerprint target_field: infoblox_bloxone_ddi.dhcp_lease.fingerprint.value ignore_missing: true - rename: + tag: rename_json_fingerprint_processed_to_infoblox_bloxone_ddi_dhcp_lease_fingerprint_processed_c2f42aca field: json.fingerprint_processed target_field: infoblox_bloxone_ddi.dhcp_lease.fingerprint.processed ignore_missing: true - rename: + tag: rename_json_ha_group_to_infoblox_bloxone_ddi_dhcp_lease_ha_group_3050e467 field: json.ha_group target_field: infoblox_bloxone_ddi.dhcp_lease.ha_group ignore_missing: true - gsub: + tag: gsub_json_hardware_596a3f78 field: json.hardware pattern: '[:.]' replacement: '-' ignore_missing: true - uppercase: + tag: uppercase_json_hardware_528ed21c field: json.hardware ignore_missing: true - rename: + tag: rename_json_hardware_to_infoblox_bloxone_ddi_dhcp_lease_hardware_9901083b field: json.hardware target_field: infoblox_bloxone_ddi.dhcp_lease.hardware ignore_missing: true - rename: + tag: rename_json_host_to_infoblox_bloxone_ddi_dhcp_lease_host_a57b956b field: json.host target_field: infoblox_bloxone_ddi.dhcp_lease.host ignore_missing: true - set: + tag: set_host_name_3e9882f6 field: host.name copy_from: infoblox_bloxone_ddi.dhcp_lease.host ignore_failure: true - append: + tag: append_related_hosts_0c50b474 field: related.hosts value: '{{{host.name}}}' if: ctx.host?.name != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_hostname_to_infoblox_bloxone_ddi_dhcp_lease_hostname_8574c513 field: json.hostname target_field: infoblox_bloxone_ddi.dhcp_lease.hostname ignore_missing: true - set: + tag: set_host_hostname_e1e135a9 field: host.hostname copy_from: infoblox_bloxone_ddi.dhcp_lease.hostname ignore_failure: true - append: + tag: append_related_hosts_e6462550 field: related.hosts value: '{{{host.hostname}}}' if: ctx.host?.hostname != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_iaid_to_infoblox_bloxone_ddi_dhcp_lease_iaid_8a6c9d6d field: json.iaid target_field: infoblox_bloxone_ddi.dhcp_lease.iaid if: ctx.json?.iaid != '' @@ -133,9 +162,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_6f5e79e8 field: error.message value: '{{{_ingest.on_failure_message}}}' - date: + tag: date_json_last_updated_to_infoblox_bloxone_ddi_dhcp_lease_last_updated_a75ac04e field: json.last_updated target_field: infoblox_bloxone_ddi.dhcp_lease.last_updated if: ctx.json?.last_updated != null && ctx.json.last_updated != '' @@ -143,17 +174,21 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_2f57298f field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_timestamp_dc7215f9 field: '@timestamp' copy_from: infoblox_bloxone_ddi.dhcp_lease.last_updated ignore_failure: true - rename: + tag: rename_json_options_to_infoblox_bloxone_ddi_dhcp_lease_options_3d08d0f3 field: json.options target_field: infoblox_bloxone_ddi.dhcp_lease.options ignore_missing: true - date: + tag: date_json_preferred_lifetime_to_infoblox_bloxone_ddi_dhcp_lease_preferred_lifetime_1a4099dc field: json.preferred_lifetime target_field: infoblox_bloxone_ddi.dhcp_lease.preferred_lifetime if: ctx.json?.preferred_lifetime != null && ctx.json.preferred_lifetime != '' @@ -161,34 +196,42 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_e3cf6031 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_json_protocol_420e2bce field: json.protocol value: ipv4 if: ctx.json?.protocol == 'ip4' ignore_failure: true - set: + tag: set_json_protocol_7448394a field: json.protocol value: ipv6 if: ctx.json?.protocol == 'ip6' ignore_failure: true - rename: + tag: rename_json_protocol_to_infoblox_bloxone_ddi_dhcp_lease_protocol_a1a8e5c3 field: json.protocol target_field: infoblox_bloxone_ddi.dhcp_lease.protocol ignore_missing: true - set: + tag: set_network_type_cf82cbe1 field: network.type copy_from: infoblox_bloxone_ddi.dhcp_lease.protocol ignore_failure: true - lowercase: + tag: lowercase_network_type_e98b3947 field: network.type ignore_failure: true - rename: + tag: rename_json_space_to_infoblox_bloxone_ddi_dhcp_lease_space_28751503 field: json.space target_field: infoblox_bloxone_ddi.dhcp_lease.space ignore_missing: true - date: + tag: date_json_starts_to_infoblox_bloxone_ddi_dhcp_lease_starts_dc90d0dc field: json.starts target_field: infoblox_bloxone_ddi.dhcp_lease.starts if: ctx.json?.starts != null && ctx.json.starts != '' @@ -196,35 +239,44 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_015288c5 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_event_start_b78cf9fc field: event.start copy_from: infoblox_bloxone_ddi.dhcp_lease.starts ignore_failure: true - rename: + tag: rename_json_state_to_infoblox_bloxone_ddi_dhcp_lease_state_874099bd field: json.state target_field: infoblox_bloxone_ddi.dhcp_lease.state ignore_missing: true - rename: + tag: rename_json_type_to_infoblox_bloxone_ddi_dhcp_lease_type_89325657 field: json.type target_field: infoblox_bloxone_ddi.dhcp_lease.type ignore_missing: true - json: + tag: json_infoblox_bloxone_ddi_dhcp_lease_options_to_infoblox_bloxone_ddi_dhcp_lease_options_2f967340 field: infoblox_bloxone_ddi.dhcp_lease.options target_field: infoblox_bloxone_ddi.dhcp_lease.options if: ctx.infoblox_bloxone_ddi?.dhcp_lease?.options instanceof String on_failure: - remove: + tag: remove_infoblox_bloxone_ddi_dhcp_lease_options_0695192a field: infoblox_bloxone_ddi.dhcp_lease.options ignore_failure: true - append: + tag: append_error_message_52a30439 field: error.message value: '{{{_ingest.on_failure_message}}}' - remove: + tag: remove_json_29cdffdc field: json ignore_missing: true - remove: + tag: remove_1f750f93 field: - infoblox_bloxone_ddi.dhcp_lease.last_updated - infoblox_bloxone_ddi.dhcp_lease.client_id @@ -237,6 +289,7 @@ processors: ignore_failure: true ignore_missing: true - script: + tag: script_e2ccbdd7 description: Drops null/empty values recursively. lang: painless source: | diff --git a/packages/infoblox_bloxone_ddi/data_stream/dns_config/elasticsearch/ingest_pipeline/default.yml b/packages/infoblox_bloxone_ddi/data_stream/dns_config/elasticsearch/ingest_pipeline/default.yml index 0e18263113f..93498e72892 100644 --- a/packages/infoblox_bloxone_ddi/data_stream/dns_config/elasticsearch/ingest_pipeline/default.yml +++ b/packages/infoblox_bloxone_ddi/data_stream/dns_config/elasticsearch/ingest_pipeline/default.yml @@ -2,18 +2,23 @@ description: Pipeline for parsing DNS config logs. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - set: + tag: set_event_type_42379782 field: event.type value: [protocol] - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -25,11 +30,14 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - drop: + tag: drop_88d6232c if: ctx.json?.results instanceof List && ctx.json.results.length == 0 - fingerprint: + tag: fingerprint_8b3026d5 fields: - json.created_at - json.updated_at @@ -37,6 +45,7 @@ processors: target_field: _id ignore_missing: true - convert: + tag: convert_json_add_edns_option_in_outgoing_query_to_infoblox_bloxone_ddi_dns_config_add_edns_option_in_outgoing_query_cfd8df9c field: json.add_edns_option_in_outgoing_query target_field: infoblox_bloxone_ddi.dns_config.add_edns.option_in.outgoing_query if: ctx.json?.add_edns_option_in_outgoing_query != '' @@ -44,13 +53,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9b932901 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_comment_to_infoblox_bloxone_ddi_dns_config_comment_664b9519 field: json.comment target_field: infoblox_bloxone_ddi.dns_config.comment ignore_missing: true - date: + tag: date_json_created_at_to_infoblox_bloxone_ddi_dns_config_created_at_76885f4e field: json.created_at target_field: infoblox_bloxone_ddi.dns_config.created_at if: ctx.json?.created_at != null && ctx.json.created_at != '' @@ -58,13 +70,16 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_bb38a6b3 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_event_created_054ceb83 field: event.created copy_from: infoblox_bloxone_ddi.dns_config.created_at ignore_failure: true - foreach: + tag: foreach_json_custom_root_ns_16c95de0 field: json.custom_root_ns if: ctx.json?.custom_root_ns instanceof List processor: @@ -82,6 +97,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_custom_root_ns_32c5cd24 field: json.custom_root_ns if: ctx.json?.custom_root_ns instanceof List processor: @@ -93,6 +109,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_custom_root_ns_93a89986 field: json.custom_root_ns if: ctx.json?.custom_root_ns instanceof List processor: @@ -103,10 +120,12 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_custom_root_ns_to_infoblox_bloxone_ddi_dns_config_custom_root_ns_46b7709b field: json.custom_root_ns target_field: infoblox_bloxone_ddi.dns_config.custom_root_ns ignore_missing: true - convert: + tag: convert_json_custom_root_ns_enabled_to_infoblox_bloxone_ddi_dns_config_custom_root_ns_enabled_72d80f46 field: json.custom_root_ns_enabled target_field: infoblox_bloxone_ddi.dns_config.custom_root_ns_enabled if: ctx.json?.custom_root_ns_enabled != '' @@ -114,9 +133,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_37be1ec9 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_disabled_to_infoblox_bloxone_ddi_dns_config_disabled_14ca4d76 field: json.disabled target_field: infoblox_bloxone_ddi.dns_config.disabled if: ctx.json?.disabled != '' @@ -124,9 +145,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f02db3b9 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_dnssec_enable_validation_to_infoblox_bloxone_ddi_dns_config_dnssec_enable_validation_af34425b field: json.dnssec_enable_validation target_field: infoblox_bloxone_ddi.dns_config.dnssec.enable_validation if: ctx.json?.dnssec_enable_validation != '' @@ -134,9 +157,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d51d8020 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_dnssec_enabled_to_infoblox_bloxone_ddi_dns_config_dnssec_enabled_8aab6c3d field: json.dnssec_enabled target_field: infoblox_bloxone_ddi.dns_config.dnssec.enabled if: ctx.json?.dnssec_enabled != '' @@ -144,9 +169,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9c19e436 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_dnssec_root_keys_60c1792d field: json.dnssec_root_keys if: ctx.json?.dnssec_root_keys instanceof List processor: @@ -163,6 +190,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_dnssec_root_keys_9a945006 field: json.dnssec_root_keys if: ctx.json?.dnssec_root_keys instanceof List processor: @@ -173,6 +201,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_dnssec_root_keys_67250f1d field: json.dnssec_root_keys if: ctx.json?.dnssec_root_keys instanceof List processor: @@ -183,6 +212,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_dnssec_root_keys_22d07101 field: json.dnssec_root_keys if: ctx.json?.dnssec_root_keys instanceof List processor: @@ -199,10 +229,12 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_dnssec_root_keys_to_infoblox_bloxone_ddi_dns_config_dnssec_root_keys_23e2bcec field: json.dnssec_root_keys target_field: infoblox_bloxone_ddi.dns_config.dnssec.root_keys ignore_missing: true - foreach: + tag: foreach_json_dnssec_trust_anchors_a51c55cd field: json.dnssec_trust_anchors if: ctx.json?.dnssec_trust_anchors instanceof List processor: @@ -219,6 +251,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_dnssec_trust_anchors_eb06fe26 field: json.dnssec_trust_anchors if: ctx.json?.dnssec_trust_anchors instanceof List processor: @@ -229,6 +262,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_dnssec_trust_anchors_658ddf21 field: json.dnssec_trust_anchors if: ctx.json?.dnssec_trust_anchors instanceof List processor: @@ -245,10 +279,12 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_dnssec_trust_anchors_to_infoblox_bloxone_ddi_dns_config_dnssec_trust_anchors_cd25e070 field: json.dnssec_trust_anchors target_field: infoblox_bloxone_ddi.dns_config.dnssec.trust_anchors ignore_missing: true - convert: + tag: convert_json_dnssec_validate_expiry_to_infoblox_bloxone_ddi_dns_config_dnssec_validate_expiry_a114afbc field: json.dnssec_validate_expiry target_field: infoblox_bloxone_ddi.dns_config.dnssec.validate_expiry if: ctx.json?.dnssec_validate_expiry != '' @@ -256,9 +292,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_90dcb377 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_ecs_enabled_to_infoblox_bloxone_ddi_dns_config_ecs_enabled_10715194 field: json.ecs_enabled target_field: infoblox_bloxone_ddi.dns_config.ecs.enabled if: ctx.json?.ecs_enabled != '' @@ -266,9 +304,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d8965a3d field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_ecs_forwarding_to_infoblox_bloxone_ddi_dns_config_ecs_forwarding_d406e420 field: json.ecs_forwarding target_field: infoblox_bloxone_ddi.dns_config.ecs.forwarding if: ctx.json?.ecs_forwarding != '' @@ -276,9 +316,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_bb69a32f field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_ecs_prefix_v4_to_infoblox_bloxone_ddi_dns_config_ecs_prefix_v4_178f9424 field: json.ecs_prefix_v4 target_field: infoblox_bloxone_ddi.dns_config.ecs.prefix_v4 if: ctx.json?.ecs_prefix_v4 != '' @@ -286,9 +328,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f0de776f field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_ecs_prefix_v6_to_infoblox_bloxone_ddi_dns_config_ecs_prefix_v6_26ce7e62 field: json.ecs_prefix_v6 target_field: infoblox_bloxone_ddi.dns_config.ecs.prefix_v6 if: ctx.json?.ecs_prefix_v6 != '' @@ -296,9 +340,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_3f032255 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_ecs_zones_1d68d022 field: json.ecs_zones if: ctx.json?.ecs_zones instanceof List processor: @@ -309,10 +355,12 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_ecs_zones_to_infoblox_bloxone_ddi_dns_config_ecs_zones_01a74eae field: json.ecs_zones target_field: infoblox_bloxone_ddi.dns_config.ecs.zones ignore_missing: true - convert: + tag: convert_json_edns_udp_size_to_infoblox_bloxone_ddi_dns_config_edns_udp_size_560f0760 field: json.edns_udp_size target_field: infoblox_bloxone_ddi.dns_config.edns.udp.size if: ctx.json?.edns_udp_size != '' @@ -320,9 +368,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_11bfe4ff field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_forwarders_6e8a22b8 field: json.forwarders if: ctx.json?.forwarders instanceof List processor: @@ -340,6 +390,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_forwarders_090d274c field: json.forwarders if: ctx.json?.forwarders instanceof List processor: @@ -351,6 +402,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_forwarders_9d0a240e field: json.forwarders if: ctx.json?.forwarders instanceof List processor: @@ -361,10 +413,12 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_forwarders_to_infoblox_bloxone_ddi_dns_config_forwarders_9298aebf field: json.forwarders target_field: infoblox_bloxone_ddi.dns_config.forwarders ignore_missing: true - convert: + tag: convert_json_forwarders_only_to_infoblox_bloxone_ddi_dns_config_forwarders_only_3b94d7ce field: json.forwarders_only target_field: infoblox_bloxone_ddi.dns_config.forwarders_only if: ctx.json?.forwarders_only != '' @@ -372,9 +426,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c1046f7b field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_gss_tsig_enabled_to_infoblox_bloxone_ddi_dns_config_gss_tsig_enabled_f88b7395 field: json.gss_tsig_enabled target_field: infoblox_bloxone_ddi.dns_config.gss_tsig_enabled if: ctx.json?.gss_tsig_enabled != '' @@ -382,29 +438,36 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2ac9d9a2 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_id_to_infoblox_bloxone_ddi_dns_config_id_cc09c187 field: json.id target_field: infoblox_bloxone_ddi.dns_config.id ignore_missing: true - set: + tag: set_event_id_cb53963d field: event.id copy_from: infoblox_bloxone_ddi.dns_config.id ignore_failure: true - rename: + tag: rename_json_inheritance_sources_add_edns_option_in_outgoing_query_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_add_edns_option_in_outgoing_query_action_47e5fc16 field: json.inheritance_sources.add_edns_option_in_outgoing_query.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.add_edns.option_in.outgoing_query.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_add_edns_option_in_outgoing_query_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_add_edns_option_in_outgoing_query_display_name_cbf7acff field: json.inheritance_sources.add_edns_option_in_outgoing_query.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.add_edns.option_in.outgoing_query.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_add_edns_option_in_outgoing_query_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_add_edns_option_in_outgoing_query_source_b46042ea field: json.inheritance_sources.add_edns_option_in_outgoing_query.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.add_edns.option_in.outgoing_query.source ignore_missing: true - convert: + tag: convert_json_inheritance_sources_add_edns_option_in_outgoing_query_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_add_edns_option_in_outgoing_query_value_2e011e4f field: json.inheritance_sources.add_edns_option_in_outgoing_query.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.add_edns.option_in.outgoing_query.value if: ctx.json?.inheritance_sources?.add_edns_option_in_outgoing_query?.value != '' @@ -412,21 +475,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5e946782 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_custom_root_ns_block_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_custom_root_ns_block_action_1017e263 field: json.inheritance_sources.custom_root_ns_block.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.custom_root_ns.block.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_custom_root_ns_block_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_custom_root_ns_block_display_name_e8bdbaf6 field: json.inheritance_sources.custom_root_ns_block.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.custom_root_ns.block.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_custom_root_ns_block_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_custom_root_ns_block_source_bbdca849 field: json.inheritance_sources.custom_root_ns_block.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.custom_root_ns.block.source ignore_missing: true - foreach: + tag: foreach_json_inheritance_sources_custom_root_ns_block_value_custom_root_ns_ea9e7d6f field: json.inheritance_sources.custom_root_ns_block.value.custom_root_ns if: ctx.json?.inheritance_sources?.custom_root_ns_block?.value?.custom_root_ns instanceof List processor: @@ -444,6 +512,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_inheritance_sources_custom_root_ns_block_value_custom_root_ns_f57041d7 field: json.inheritance_sources.custom_root_ns_block.value.custom_root_ns if: ctx.json?.inheritance_sources?.custom_root_ns_block?.value?.custom_root_ns instanceof List processor: @@ -455,6 +524,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_inheritance_sources_custom_root_ns_block_value_custom_root_ns_8461432d field: json.inheritance_sources.custom_root_ns_block.value.custom_root_ns if: ctx.json?.inheritance_sources?.custom_root_ns_block?.value?.custom_root_ns instanceof List processor: @@ -465,10 +535,12 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_inheritance_sources_custom_root_ns_block_value_custom_root_ns_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_custom_root_ns_block_value_728d3707 field: json.inheritance_sources.custom_root_ns_block.value.custom_root_ns target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.custom_root_ns.block.value ignore_missing: true - convert: + tag: convert_json_inheritance_sources_custom_root_ns_block_value_custom_root_ns_enabled_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_custom_root_ns_block_value_enabled_5a8401fd field: json.inheritance_sources.custom_root_ns_block.value.custom_root_ns_enabled target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.custom_root_ns.block.value_enabled if: ctx.json?.inheritance_sources?.custom_root_ns_block?.value?.custom_root_ns_enabled != '' @@ -476,21 +548,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_cd1a1598 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_dnssec_validation_block_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_dnssec_validation_block_action_3d286ac8 field: json.inheritance_sources.dnssec_validation_block.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.dnssec.validation.block.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_dnssec_validation_block_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_dnssec_validation_block_display_name_d1b6a759 field: json.inheritance_sources.dnssec_validation_block.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.dnssec.validation.block.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_dnssec_validation_block_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_dnssec_validation_block_source_b9a6efd4 field: json.inheritance_sources.dnssec_validation_block.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.dnssec.validation.block.source ignore_missing: true - convert: + tag: convert_json_inheritance_sources_dnssec_validation_block_value_dnssec_enable_validation_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_dnssec_validation_block_value_enable_6254e4e7 field: json.inheritance_sources.dnssec_validation_block.value.dnssec_enable_validation target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.dnssec.validation.block.value.enable if: ctx.json?.inheritance_sources?.dnssec_validation_block?.value?.dnssec_enable_validation != '' @@ -498,9 +575,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8a060ba6 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_inheritance_sources_dnssec_validation_block_value_dnssec_enabled_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_dnssec_validation_block_value_enabled_cde1d679 field: json.inheritance_sources.dnssec_validation_block.value.dnssec_enabled target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.dnssec.validation.block.value.enabled if: ctx.json?.inheritance_sources?.dnssec_validation_block?.value?.dnssec_enabled != '' @@ -508,9 +587,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_aa547d2e field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_inheritance_sources_dnssec_validation_block_value_dnssec_trust_anchors_f87b1842 field: json.inheritance_sources.dnssec_validation_block.value.dnssec_trust_anchors if: ctx.json?.inheritance_sources?.dnssec_validation_block?.value?.dnssec_trust_anchors instanceof List processor: @@ -527,6 +608,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_inheritance_sources_dnssec_validation_block_value_dnssec_trust_anchors_7eda48ed field: json.inheritance_sources.dnssec_validation_block.value.dnssec_trust_anchors if: ctx.json?.inheritance_sources?.dnssec_validation_block?.value?.dnssec_trust_anchors instanceof List processor: @@ -537,6 +619,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_inheritance_sources_dnssec_validation_block_value_dnssec_trust_anchors_6d576f28 field: json.inheritance_sources.dnssec_validation_block.value.dnssec_trust_anchors if: ctx.json?.inheritance_sources?.dnssec_validation_block?.value?.dnssec_trust_anchors instanceof List processor: @@ -553,10 +636,12 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_inheritance_sources_dnssec_validation_block_value_dnssec_trust_anchors_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_dnssec_validation_block_value_trust_anchors_2e2e0445 field: json.inheritance_sources.dnssec_validation_block.value.dnssec_trust_anchors target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.dnssec.validation.block.value.trust_anchors ignore_missing: true - convert: + tag: convert_json_inheritance_sources_dnssec_validation_block_value_dnssec_validate_expiry_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_dnssec_validation_block_value_validate_expiry_90414042 field: json.inheritance_sources.dnssec_validation_block.value.dnssec_validate_expiry target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.dnssec.validation.block.value.validate_expiry if: ctx.json?.inheritance_sources?.dnssec_validation_block?.value?.dnssec_validate_expiry != '' @@ -564,21 +649,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_42db1039 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_ecs_block_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_ecs_block_action_9f817ddb field: json.inheritance_sources.ecs_block.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.ecs.block.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_ecs_block_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_ecs_block_display_name_f29cffb2 field: json.inheritance_sources.ecs_block.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.ecs.block.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_ecs_block_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_ecs_block_source_74c56327 field: json.inheritance_sources.ecs_block.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.ecs.block.source ignore_missing: true - convert: + tag: convert_json_inheritance_sources_ecs_block_value_ecs_enabled_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_ecs_block_value_enabled_3374d25d field: json.inheritance_sources.ecs_block.value.ecs_enabled target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.ecs.block.value.enabled if: ctx.json?.inheritance_sources?.ecs_block?.value?.ecs_enabled != '' @@ -586,9 +676,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_98475106 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_inheritance_sources_ecs_block_value_ecs_forwarding_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_ecs_block_value_forwarding_dae5d97f field: json.inheritance_sources.ecs_block.value.ecs_forwarding target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.ecs.block.value.forwarding if: ctx.json?.inheritance_sources?.ecs_block?.value?.ecs_forwarding != '' @@ -596,9 +688,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f74ccffe field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_inheritance_sources_ecs_block_value_ecs_prefix_v4_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_ecs_block_value_prefix_v4_37f5e5b3 field: json.inheritance_sources.ecs_block.value.ecs_prefix_v4 target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.ecs.block.value.prefix_v4 if: ctx.json?.inheritance_sources?.ecs_block?.value?.ecs_prefix_v4 != '' @@ -606,9 +700,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_61f7603a field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_inheritance_sources_ecs_block_value_ecs_prefix_v6_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_ecs_block_value_prefix_v6_692e141d field: json.inheritance_sources.ecs_block.value.ecs_prefix_v6 target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.ecs.block.value.prefix_v6 if: ctx.json?.inheritance_sources?.ecs_block?.value?.ecs_prefix_v6 != '' @@ -616,9 +712,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_144e9b54 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_inheritance_sources_ecs_block_value_ecs_zones_e9e1b579 field: json.inheritance_sources.ecs_block.value.ecs_zones if: ctx.json?.inheritance_sources?.ecs_block?.value?.ecs_zones instanceof List processor: @@ -629,22 +727,27 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_inheritance_sources_ecs_block_value_ecs_zones_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_ecs_block_value_zones_05480151 field: json.inheritance_sources.ecs_block.value.ecs_zones target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.ecs.block.value.zones ignore_missing: true - rename: + tag: rename_json_inheritance_sources_edns_udp_size_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_edns_udp_size_action_94b06922 field: json.inheritance_sources.edns_udp_size.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.edns.udp.size.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_edns_udp_size_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_edns_udp_size_display_name_abe97baf field: json.inheritance_sources.edns_udp_size.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.edns.udp.size.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_edns_udp_size_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_edns_udp_size_source_3cd4b0a6 field: json.inheritance_sources.edns_udp_size.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.edns.udp.size.source ignore_missing: true - convert: + tag: convert_json_inheritance_sources_edns_udp_size_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_edns_udp_size_value_1091f10d field: json.inheritance_sources.edns_udp_size.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.edns.udp.size.value if: ctx.json?.inheritance_sources?.edns_udp_size?.value != '' @@ -652,21 +755,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5d75402a field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_forwarders_block_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_forwarders_block_action_0ddc7635 field: json.inheritance_sources.forwarders_block.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.forwarders.block.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_forwarders_block_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_forwarders_block_display_name_9a43f61c field: json.inheritance_sources.forwarders_block.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.forwarders.block.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_forwarders_block_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_forwarders_block_source_59a722fb field: json.inheritance_sources.forwarders_block.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.forwarders.block.source ignore_missing: true - foreach: + tag: foreach_json_inheritance_sources_forwarders_block_value_forwarders_74bdaf7f field: json.inheritance_sources.forwarders_block.value.forwarders if: ctx.json?.inheritance_sources?.forwarders_block?.value?.forwarders instanceof List processor: @@ -684,6 +792,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_inheritance_sources_forwarders_block_value_forwarders_b91af167 field: json.inheritance_sources.forwarders_block.value.forwarders if: ctx.json?.inheritance_sources?.forwarders_block?.value?.forwarders instanceof List processor: @@ -695,6 +804,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_inheritance_sources_forwarders_block_value_forwarders_c8e72c7d field: json.inheritance_sources.forwarders_block.value.forwarders if: ctx.json?.inheritance_sources?.forwarders_block?.value?.forwarders instanceof List processor: @@ -705,10 +815,12 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_inheritance_sources_forwarders_block_value_forwarders_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_forwarders_block_value_7ca1ce2e field: json.inheritance_sources.forwarders_block.value.forwarders target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.forwarders.block.value ignore_missing: true - convert: + tag: convert_json_inheritance_sources_forwarders_block_value_forwarders_only_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_forwarders_block_value_only_4f4be407 field: json.inheritance_sources.forwarders_block.value.forwarders_only target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.forwarders.block.value_only if: ctx.json?.inheritance_sources?.forwarders_block?.value?.forwarders_only != '' @@ -716,21 +828,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e97d1414 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_gss_tsig_enabled_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_gss_tsig_enabled_action_edda5d68 field: json.inheritance_sources.gss_tsig_enabled.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.gss_tsig_enabled.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_gss_tsig_enabled_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_gss_tsig_enabled_display_name_df724985 field: json.inheritance_sources.gss_tsig_enabled.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.gss_tsig_enabled.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_gss_tsig_enabled_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_gss_tsig_enabled_source_89dc1b12 field: json.inheritance_sources.gss_tsig_enabled.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.gss_tsig_enabled.source ignore_missing: true - convert: + tag: convert_json_inheritance_sources_gss_tsig_enabled_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_gss_tsig_enabled_value_ce3a2432 field: json.inheritance_sources.gss_tsig_enabled.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.gss_tsig_enabled.value if: ctx.json?.inheritance_sources?.gss_tsig_enabled?.value != '' @@ -738,21 +855,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a48dfad9 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_lame_ttl_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_lame_ttl_action_ffe2aff6 field: json.inheritance_sources.lame_ttl.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.lame_ttl.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_lame_ttl_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_lame_ttl_display_name_b00c5ec3 field: json.inheritance_sources.lame_ttl.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.lame_ttl.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_lame_ttl_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_lame_ttl_source_a22ebcf0 field: json.inheritance_sources.lame_ttl.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.lame_ttl.source ignore_missing: true - convert: + tag: convert_json_inheritance_sources_lame_ttl_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_lame_ttl_value_13a30b21 field: json.inheritance_sources.lame_ttl.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.lame_ttl.value if: ctx.json?.inheritance_sources?.lame_ttl?.value != '' @@ -760,21 +882,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_108b5cb8 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_match_recursive_only_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_match_recursive_only_action_d510d0d8 field: json.inheritance_sources.match_recursive_only.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.match_recursive_only.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_match_recursive_only_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_match_recursive_only_display_name_e3a10d21 field: json.inheritance_sources.match_recursive_only.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.match_recursive_only.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_match_recursive_only_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_match_recursive_only_source_52fd4612 field: json.inheritance_sources.match_recursive_only.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.match_recursive_only.source ignore_missing: true - convert: + tag: convert_json_inheritance_sources_match_recursive_only_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_match_recursive_only_value_ec306aac field: json.inheritance_sources.match_recursive_only.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.match_recursive_only.value if: ctx.json?.inheritance_sources?.match_recursive_only?.value != '' @@ -782,21 +909,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f1d55073 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_max_cache_ttl_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_max_cache_ttl_action_ca4ce312 field: json.inheritance_sources.max_cache_ttl.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.max_cache_ttl.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_max_cache_ttl_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_max_cache_ttl_display_name_43017937 field: json.inheritance_sources.max_cache_ttl.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.max_cache_ttl.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_max_cache_ttl_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_max_cache_ttl_source_0fcd3552 field: json.inheritance_sources.max_cache_ttl.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.max_cache_ttl.source ignore_missing: true - convert: + tag: convert_json_inheritance_sources_max_cache_ttl_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_max_cache_ttl_value_ce887653 field: json.inheritance_sources.max_cache_ttl.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.max_cache_ttl.value if: ctx.json?.inheritance_sources?.max_cache_ttl?.value != '' @@ -804,21 +936,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_939e44d8 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_max_negative_ttl_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_max_negative_ttl_action_6b2bae84 field: json.inheritance_sources.max_negative_ttl.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.max_negative_ttl.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_max_negative_ttl_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_max_negative_ttl_display_name_da7064b1 field: json.inheritance_sources.max_negative_ttl.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.max_negative_ttl.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_max_negative_ttl_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_max_negative_ttl_source_84c51592 field: json.inheritance_sources.max_negative_ttl.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.max_negative_ttl.source ignore_missing: true - convert: + tag: convert_json_inheritance_sources_max_negative_ttl_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_max_negative_ttl_value_05cedae4 field: json.inheritance_sources.max_negative_ttl.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.max_negative_ttl.value if: ctx.json?.inheritance_sources?.max_negative_ttl?.value != '' @@ -826,21 +963,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_21513d91 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_max_udp_size_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_max_udp_size_action_d5a6631a field: json.inheritance_sources.max_udp_size.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.max_udp_size.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_max_udp_size_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_max_udp_size_display_name_38a16e43 field: json.inheritance_sources.max_udp_size.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.max_udp_size.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_max_udp_size_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_max_udp_size_source_7aaab644 field: json.inheritance_sources.max_udp_size.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.max_udp_size.source ignore_missing: true - convert: + tag: convert_json_inheritance_sources_max_udp_size_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_max_udp_size_value_865cfea9 field: json.inheritance_sources.max_udp_size.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.max_udp_size.value if: ctx.json?.inheritance_sources?.max_udp_size?.value != '' @@ -848,21 +990,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ec4610f4 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_minimal_responses_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_minimal_responses_action_f5f96aaa field: json.inheritance_sources.minimal_responses.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.minimal_responses.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_minimal_responses_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_minimal_responses_display_name_5d29267f field: json.inheritance_sources.minimal_responses.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.minimal_responses.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_minimal_responses_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_minimal_responses_source_180ea67a field: json.inheritance_sources.minimal_responses.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.minimal_responses.source ignore_missing: true - convert: + tag: convert_json_inheritance_sources_minimal_responses_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_minimal_responses_value_cdab6e13 field: json.inheritance_sources.minimal_responses.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.minimal_responses.value if: ctx.json?.inheritance_sources?.minimal_responses?.value != '' @@ -870,21 +1017,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_fc3a0b72 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_notify_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_notify_action_419db1ac field: json.inheritance_sources.notify.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.notify.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_notify_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_notify_display_name_86727331 field: json.inheritance_sources.notify.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.notify.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_notify_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_notify_source_32d0de12 field: json.inheritance_sources.notify.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.notify.source ignore_missing: true - convert: + tag: convert_json_inheritance_sources_notify_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_notify_value_03852eb4 field: json.inheritance_sources.notify.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.notify.value if: ctx.json?.inheritance_sources?.notify?.value != '' @@ -892,21 +1044,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_0c17b4e3 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_query_acl_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_query_acl_action_fe4a3972 field: json.inheritance_sources.query_acl.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.query_acl.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_query_acl_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_query_acl_display_name_d251d323 field: json.inheritance_sources.query_acl.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.query_acl.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_query_acl_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_query_acl_source_9d828f5e field: json.inheritance_sources.query_acl.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.query_acl.source ignore_missing: true - foreach: + tag: foreach_json_inheritance_sources_query_acl_value_d63342f0 field: json.inheritance_sources.query_acl.value if: ctx.json?.inheritance_sources?.query_acl?.value instanceof List processor: @@ -924,6 +1081,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_inheritance_sources_query_acl_value_fd634d34 field: json.inheritance_sources.query_acl.value if: ctx.json?.inheritance_sources?.query_acl?.value instanceof List processor: @@ -935,6 +1093,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_inheritance_sources_query_acl_value_6affcc47 field: json.inheritance_sources.query_acl.value if: ctx.json?.inheritance_sources?.query_acl?.value instanceof List processor: @@ -946,6 +1105,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_inheritance_sources_query_acl_value_5cf64690 field: json.inheritance_sources.query_acl.value if: ctx.json?.inheritance_sources?.query_acl?.value instanceof List processor: @@ -956,22 +1116,27 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_inheritance_sources_query_acl_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_query_acl_value_4fae1e92 field: json.inheritance_sources.query_acl.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.query_acl.value ignore_missing: true - rename: + tag: rename_json_inheritance_sources_recursion_acl_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_recursion_acl_action_905e4ab2 field: json.inheritance_sources.recursion_acl.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.recursion_acl.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_recursion_acl_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_recursion_acl_display_name_b17c2e87 field: json.inheritance_sources.recursion_acl.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.recursion_acl.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_recursion_acl_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_recursion_acl_source_41e37542 field: json.inheritance_sources.recursion_acl.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.recursion_acl.source ignore_missing: true - foreach: + tag: foreach_json_inheritance_sources_recursion_acl_value_9036962c field: json.inheritance_sources.recursion_acl.value if: ctx.json?.inheritance_sources?.recursion_acl?.value instanceof List processor: @@ -989,6 +1154,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_inheritance_sources_recursion_acl_value_4bb9e420 field: json.inheritance_sources.recursion_acl.value if: ctx.json?.inheritance_sources?.recursion_acl?.value instanceof List processor: @@ -1000,6 +1166,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_inheritance_sources_recursion_acl_value_b9a61cf3 field: json.inheritance_sources.recursion_acl.value if: ctx.json?.inheritance_sources?.recursion_acl?.value instanceof List processor: @@ -1011,6 +1178,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_inheritance_sources_recursion_acl_value_0f34d6b4 field: json.inheritance_sources.recursion_acl.value if: ctx.json?.inheritance_sources?.recursion_acl?.value instanceof List processor: @@ -1021,22 +1189,27 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_inheritance_sources_recursion_acl_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_recursion_acl_value_91f52552 field: json.inheritance_sources.recursion_acl.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.recursion_acl.value ignore_missing: true - rename: + tag: rename_json_inheritance_sources_recursion_enabled_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_recursion_enabled_action_4d547902 field: json.inheritance_sources.recursion_enabled.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.recursion_enabled.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_recursion_enabled_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_recursion_enabled_display_name_a0c1b813 field: json.inheritance_sources.recursion_enabled.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.recursion_enabled.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_recursion_enabled_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_recursion_enabled_source_0819519e field: json.inheritance_sources.recursion_enabled.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.recursion_enabled.source ignore_missing: true - convert: + tag: convert_json_inheritance_sources_recursion_enabled_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_recursion_enabled_value_d35888c9 field: json.inheritance_sources.recursion_enabled.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.recursion_enabled.value if: ctx.json?.inheritance_sources?.recursion_enabled?.value != '' @@ -1044,21 +1217,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_66c71810 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_synthesize_address_records_from_https_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_synthesize_address_records_from_https_action_72084061 field: json.inheritance_sources.synthesize_address_records_from_https.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.synthesize.address_records_from_https.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_synthesize_address_records_from_https_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_synthesize_address_records_from_https_display_name_5c7e36b4 field: json.inheritance_sources.synthesize_address_records_from_https.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.synthesize.address_records_from_https.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_synthesize_address_records_from_https_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_synthesize_address_records_from_https_name_a9737923 field: json.inheritance_sources.synthesize_address_records_from_https.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.synthesize.address_records_from_https.name ignore_missing: true - convert: + tag: convert_json_inheritance_sources_synthesize_address_records_from_https_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_synthesize_address_records_from_https_value_b198e1a5 field: json.inheritance_sources.synthesize_address_records_from_https.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.synthesize.address_records_from_https.value if: ctx.json?.inheritance_sources?.synthesize_address_records_from_https?.value != '' @@ -1066,21 +1244,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_b97613d8 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_transfer_acl_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_transfer_acl_action_3bce2e7a field: json.inheritance_sources.transfer_acl.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.transfer_acl.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_transfer_acl_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_transfer_acl_display_name_15685ed7 field: json.inheritance_sources.transfer_acl.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.transfer_acl.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_transfer_acl_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_transfer_acl_source_14fe6a3c field: json.inheritance_sources.transfer_acl.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.transfer_acl.source ignore_missing: true - foreach: + tag: foreach_json_inheritance_sources_transfer_acl_value_7b5dc79c field: json.inheritance_sources.transfer_acl.value if: ctx.json?.inheritance_sources?.transfer_acl?.value instanceof List processor: @@ -1098,6 +1281,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_inheritance_sources_transfer_acl_value_3a5b5890 field: json.inheritance_sources.transfer_acl.value if: ctx.json?.inheritance_sources?.transfer_acl?.value instanceof List processor: @@ -1109,6 +1293,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_inheritance_sources_transfer_acl_value_847505a3 field: json.inheritance_sources.transfer_acl.value if: ctx.json?.inheritance_sources?.transfer_acl?.value instanceof List processor: @@ -1120,6 +1305,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_inheritance_sources_transfer_acl_value_8a4ed3a4 field: json.inheritance_sources.transfer_acl.value if: ctx.json?.inheritance_sources?.transfer_acl?.value instanceof List processor: @@ -1130,22 +1316,27 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_inheritance_sources_transfer_acl_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_transfer_acl_value_70b9625a field: json.inheritance_sources.transfer_acl.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.transfer_acl.value ignore_missing: true - rename: + tag: rename_json_inheritance_sources_update_acl_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_update_acl_action_b92850e2 field: json.inheritance_sources.update_acl.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.update_acl.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_update_acl_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_update_acl_display_name_fbd98023 field: json.inheritance_sources.update_acl.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.update_acl.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_update_acl_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_update_acl_source_369e60bc field: json.inheritance_sources.update_acl.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.update_acl.source ignore_missing: true - foreach: + tag: foreach_json_inheritance_sources_update_acl_value_f3c7cb88 field: json.inheritance_sources.update_acl.value if: ctx.json?.inheritance_sources?.update_acl?.value instanceof List processor: @@ -1163,6 +1354,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_inheritance_sources_update_acl_value_5555069c field: json.inheritance_sources.update_acl.value if: ctx.json?.inheritance_sources?.update_acl?.value instanceof List processor: @@ -1174,6 +1366,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_inheritance_sources_update_acl_value_4f91ec2f field: json.inheritance_sources.update_acl.value if: ctx.json?.inheritance_sources?.update_acl?.value instanceof List processor: @@ -1185,6 +1378,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_inheritance_sources_update_acl_value_9e3cc758 field: json.inheritance_sources.update_acl.value if: ctx.json?.inheritance_sources?.update_acl?.value instanceof List processor: @@ -1195,22 +1389,27 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_inheritance_sources_update_acl_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_update_acl_value_c204ea9a field: json.inheritance_sources.update_acl.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.update_acl.value ignore_missing: true - rename: + tag: rename_json_inheritance_sources_use_forwarders_for_subzones_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_use_forwarders_for_subzones_action_5f220696 field: json.inheritance_sources.use_forwarders_for_subzones.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.use_forwarders_for_subzones.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_use_forwarders_for_subzones_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_use_forwarders_for_subzones_display_name_8a63ded3 field: json.inheritance_sources.use_forwarders_for_subzones.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.use_forwarders_for_subzones.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_use_forwarders_for_subzones_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_use_forwarders_for_subzones_source_7b34d962 field: json.inheritance_sources.use_forwarders_for_subzones.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.use_forwarders_for_subzones.source ignore_missing: true - convert: + tag: convert_json_inheritance_sources_use_forwarders_for_subzones_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_use_forwarders_for_subzones_value_08e13442 field: json.inheritance_sources.use_forwarders_for_subzones.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.use_forwarders_for_subzones.value if: ctx.json?.inheritance_sources?.use_forwarders_for_subzones?.value != '' @@ -1218,21 +1417,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d88171d7 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_zone_authority_default_ttl_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_default_ttl_action_073388d2 field: json.inheritance_sources.zone_authority.default_ttl.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.default_ttl.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_default_ttl_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_default_ttl_display_name_498c918f field: json.inheritance_sources.zone_authority.default_ttl.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.default_ttl.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_default_ttl_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_default_ttl_source_fd180dd0 field: json.inheritance_sources.zone_authority.default_ttl.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.default_ttl.source ignore_missing: true - convert: + tag: convert_json_inheritance_sources_zone_authority_default_ttl_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_default_ttl_value_1b91d5a2 field: json.inheritance_sources.zone_authority.default_ttl.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.default_ttl.value if: ctx.json?.inheritance_sources?.zone_authority?.default_ttl?.value != '' @@ -1240,21 +1444,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_62150c1d field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_zone_authority_expire_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_expire_action_690d0c36 field: json.inheritance_sources.zone_authority.expire.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.expire.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_expire_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_expire_display_name_f5dcda2b field: json.inheritance_sources.zone_authority.expire.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.expire.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_expire_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_expire_source_bd79858a field: json.inheritance_sources.zone_authority.expire.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.expire.source ignore_missing: true - convert: + tag: convert_json_inheritance_sources_zone_authority_expire_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_expire_value_54c63cf3 field: json.inheritance_sources.zone_authority.expire.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.expire.value if: ctx.json?.inheritance_sources?.zone_authority?.expire?.value != '' @@ -1262,29 +1471,36 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a10edc8e field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_zone_authority_mname_block_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_mname_block_action_7dcfe052 field: json.inheritance_sources.zone_authority.mname_block.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.mname_block.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_mname_block_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_mname_block_display_name_94bb53ef field: json.inheritance_sources.zone_authority.mname_block.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.mname_block.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_mname_block_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_mname_block_source_3589c338 field: json.inheritance_sources.zone_authority.mname_block.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.mname_block.source ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_mname_block_value_mname_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_mname_block_value_733ccf93 field: json.inheritance_sources.zone_authority.mname_block.value.mname target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.mname_block_value ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_mname_block_value_protocol_mname_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_mname_block_value_protocol_mname_af7fbe1d field: json.inheritance_sources.zone_authority.mname_block.value.protocol_mname target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.mname_block.value.protocol.mname ignore_missing: true - convert: + tag: convert_json_inheritance_sources_zone_authority_mname_block_value_use_default_mname_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_mname_block_value_isdefault_9e1e0684 field: json.inheritance_sources.zone_authority.mname_block.value.use_default_mname target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.mname_block.value.isdefault if: ctx.json?.inheritance_sources?.zone_authority?.mname_block?.value?.use_default_mname != '' @@ -1292,21 +1508,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_56d20a5f field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_zone_authority_negative_ttl_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_negative_ttl_action_3349e242 field: json.inheritance_sources.zone_authority.negative_ttl.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.negative_ttl.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_negative_ttl_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_negative_ttl_display_name_0b88e8e3 field: json.inheritance_sources.zone_authority.negative_ttl.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.negative_ttl.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_negative_ttl_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_negative_ttl_source_f1b23932 field: json.inheritance_sources.zone_authority.negative_ttl.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.negative_ttl.source ignore_missing: true - convert: + tag: convert_json_inheritance_sources_zone_authority_negative_ttl_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_negative_ttl_value_81224f7c field: json.inheritance_sources.zone_authority.negative_ttl.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.negative_ttl.value if: ctx.json?.inheritance_sources?.zone_authority?.negative_ttl?.value != '' @@ -1314,37 +1535,46 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_89b6ad89 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_zone_authority_protocol_rname_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_protocol_rname_action_b5934b72 field: json.inheritance_sources.zone_authority.protocol_rname.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.protocol_rname.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_protocol_rname_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_protocol_rname_display_name_cc09da8b field: json.inheritance_sources.zone_authority.protocol_rname.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.protocol_rname.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_protocol_rname_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_protocol_rname_source_b543e766 field: json.inheritance_sources.zone_authority.protocol_rname.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.protocol_rname.source ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_protocol_rname_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_protocol_rname_value_7f87ceac field: json.inheritance_sources.zone_authority.protocol_rname.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.protocol_rname.value ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_refresh_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_refresh_action_2c3ecd98 field: json.inheritance_sources.zone_authority.refresh.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.refresh.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_refresh_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_refresh_display_name_4438261d field: json.inheritance_sources.zone_authority.refresh.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.refresh.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_refresh_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_refresh_source_0a886952 field: json.inheritance_sources.zone_authority.refresh.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.refresh.source ignore_missing: true - convert: + tag: convert_json_inheritance_sources_zone_authority_refresh_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_refresh_value_69bfe8e9 field: json.inheritance_sources.zone_authority.refresh.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.refresh.value if: ctx.json?.inheritance_sources?.zone_authority?.refresh?.value != '' @@ -1352,21 +1582,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e9e473e2 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_zone_authority_retry_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_retry_action_fe7a8532 field: json.inheritance_sources.zone_authority.retry.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.retry.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_retry_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_retry_display_name_efbf2e07 field: json.inheritance_sources.zone_authority.retry.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.retry.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_retry_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_retry_source_1812e8d8 field: json.inheritance_sources.zone_authority.retry.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.retry.source ignore_missing: true - convert: + tag: convert_json_inheritance_sources_zone_authority_retry_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_retry_value_5a617726 field: json.inheritance_sources.zone_authority.retry.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.retry.value if: ctx.json?.inheritance_sources?.zone_authority?.retry?.value != '' @@ -1374,29 +1609,36 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_94adb4e1 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_inheritance_sources_zone_authority_rname_action_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_rname_action_99f225c4 field: json.inheritance_sources.zone_authority.rname.action target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.rname.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_rname_display_name_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_rname_display_name_e617b2f1 field: json.inheritance_sources.zone_authority.rname.display_name target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.rname.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_rname_source_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_rname_source_36ea9052 field: json.inheritance_sources.zone_authority.rname.source target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.rname.source ignore_missing: true - rename: + tag: rename_json_inheritance_sources_zone_authority_rname_value_to_infoblox_bloxone_ddi_dns_config_inheritance_sources_zone_authority_rname_value_2b6c867a field: json.inheritance_sources.zone_authority.rname.value target_field: infoblox_bloxone_ddi.dns_config.inheritance.sources.zone_authority.rname.value ignore_missing: true - rename: + tag: rename_json_ip_spaces_to_infoblox_bloxone_ddi_dns_config_ip_spaces_4dc001a9 field: json.ip_spaces target_field: infoblox_bloxone_ddi.dns_config.ip_spaces ignore_missing: true - convert: + tag: convert_json_lame_ttl_to_infoblox_bloxone_ddi_dns_config_lame_ttl_afaeb692 field: json.lame_ttl target_field: infoblox_bloxone_ddi.dns_config.lame_ttl if: ctx.json?.lame_ttl != '' @@ -1404,13 +1646,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_bcab46a3 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_dns_answers_ttl_209b6eeb field: dns.answers.ttl copy_from: infoblox_bloxone_ddi.dns_config.lame_ttl ignore_failure: true - foreach: + tag: foreach_json_match_clients_acl_a0ba064c field: json.match_clients_acl if: ctx.json?.match_clients_acl instanceof List processor: @@ -1428,6 +1673,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_match_clients_acl_a0f89d00 field: json.match_clients_acl if: ctx.json?.match_clients_acl instanceof List processor: @@ -1439,6 +1685,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_match_clients_acl_a36368e4 field: json.match_clients_acl if: ctx.json?.match_clients_acl instanceof List processor: @@ -1449,6 +1696,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_match_clients_acl_ccd94d93 field: json.match_clients_acl if: ctx.json?.match_clients_acl instanceof List processor: @@ -1460,6 +1708,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_match_clients_acl_bb4d7e54 field: json.match_clients_acl if: ctx.json?.match_clients_acl instanceof List processor: @@ -1470,10 +1719,12 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_match_clients_acl_to_infoblox_bloxone_ddi_dns_config_match_clients_acl_d335ee79 field: json.match_clients_acl target_field: infoblox_bloxone_ddi.dns_config.match_clients_acl ignore_missing: true - foreach: + tag: foreach_json_match_destinations_acl_8b261a28 field: json.match_destinations_acl if: ctx.json?.match_destinations_acl instanceof List processor: @@ -1491,6 +1742,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_match_destinations_acl_99c9d97c field: json.match_destinations_acl if: ctx.json?.match_destinations_acl instanceof List processor: @@ -1502,6 +1754,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_match_destinations_acl_d1faf220 field: json.match_destinations_acl if: ctx.json?.match_destinations_acl instanceof List processor: @@ -1512,6 +1765,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_match_destinations_acl_2aba51cf field: json.match_destinations_acl if: ctx.json?.match_destinations_acl instanceof List processor: @@ -1523,6 +1777,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_match_destinations_acl_ae1205f8 field: json.match_destinations_acl if: ctx.json?.match_destinations_acl instanceof List processor: @@ -1533,10 +1788,12 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_match_destinations_acl_to_infoblox_bloxone_ddi_dns_config_match_destinations_acl_571e4b87 field: json.match_destinations_acl target_field: infoblox_bloxone_ddi.dns_config.match_destinations_acl ignore_missing: true - convert: + tag: convert_json_match_recursive_only_to_infoblox_bloxone_ddi_dns_config_match_recursive_only_fcdfb91f field: json.match_recursive_only target_field: infoblox_bloxone_ddi.dns_config.match_recursive_only if: ctx.json?.match_recursive_only != '' @@ -1544,9 +1801,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a667e508 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_max_cache_ttl_to_infoblox_bloxone_ddi_dns_config_max_cache_ttl_6c10fad2 field: json.max_cache_ttl target_field: infoblox_bloxone_ddi.dns_config.max_cache_ttl if: ctx.json?.max_cache_ttl != '' @@ -1554,9 +1813,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_799eb681 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_max_negative_ttl_to_infoblox_bloxone_ddi_dns_config_max_negative_ttl_a8d50275 field: json.max_negative_ttl target_field: infoblox_bloxone_ddi.dns_config.max_negative_ttl if: ctx.json?.max_negative_ttl != '' @@ -1564,9 +1825,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2b0eff24 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_max_udp_size_to_infoblox_bloxone_ddi_dns_config_max_udp_size_e41a533a field: json.max_udp_size target_field: infoblox_bloxone_ddi.dns_config.max_udp_size if: ctx.json?.max_udp_size != '' @@ -1574,9 +1837,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_99706e4b field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_minimal_responses_to_infoblox_bloxone_ddi_dns_config_minimal_responses_f2b8a7a4 field: json.minimal_responses target_field: infoblox_bloxone_ddi.dns_config.minimal_responses if: ctx.json?.minimal_responses != '' @@ -1584,13 +1849,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_52fa0c39 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_name_to_infoblox_bloxone_ddi_dns_config_name_102088c7 field: json.name target_field: infoblox_bloxone_ddi.dns_config.name ignore_missing: true - convert: + tag: convert_json_notify_to_infoblox_bloxone_ddi_dns_config_notify_b9485fa3 field: json.notify target_field: infoblox_bloxone_ddi.dns_config.notify if: ctx.json?.notify != '' @@ -1598,9 +1866,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_7a50f120 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_query_acl_a1ce3aa0 field: json.query_acl if: ctx.json?.query_acl instanceof List processor: @@ -1618,6 +1888,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_query_acl_a64d6f64 field: json.query_acl if: ctx.json?.query_acl instanceof List processor: @@ -1629,6 +1900,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_query_acl_189106f7 field: json.query_acl if: ctx.json?.query_acl instanceof List processor: @@ -1640,6 +1912,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_query_acl_13727628 field: json.query_acl if: ctx.json?.query_acl instanceof List processor: @@ -1650,6 +1923,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_query_acl_f772ca00 field: json.query_acl if: ctx.json?.query_acl instanceof List processor: @@ -1660,10 +1934,12 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_query_acl_to_infoblox_bloxone_ddi_dns_config_query_acl_8f779129 field: json.query_acl target_field: infoblox_bloxone_ddi.dns_config.query_acl ignore_missing: true - foreach: + tag: foreach_json_recursion_acl_024c7b30 field: json.recursion_acl if: ctx.json?.recursion_acl instanceof List processor: @@ -1681,6 +1957,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_recursion_acl_790cf3f4 field: json.recursion_acl if: ctx.json?.recursion_acl instanceof List processor: @@ -1692,6 +1969,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_recursion_acl_b9f34187 field: json.recursion_acl if: ctx.json?.recursion_acl instanceof List processor: @@ -1703,6 +1981,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_recursion_acl_60a18398 field: json.recursion_acl if: ctx.json?.recursion_acl instanceof List processor: @@ -1713,6 +1992,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_recursion_acl_bfb60150 field: json.recursion_acl if: ctx.json?.recursion_acl instanceof List processor: @@ -1723,10 +2003,12 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_recursion_acl_to_infoblox_bloxone_ddi_dns_config_recursion_acl_b45ce4b5 field: json.recursion_acl target_field: infoblox_bloxone_ddi.dns_config.recursion_acl ignore_missing: true - convert: + tag: convert_json_recursion_enabled_to_infoblox_bloxone_ddi_dns_config_recursion_enabled_94b5b6f6 field: json.recursion_enabled target_field: infoblox_bloxone_ddi.dns_config.recursion_enabled if: ctx.json?.recursion_enabled != '' @@ -1734,9 +2016,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_1e629bb7 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_synthesize_address_records_from_https_to_infoblox_bloxone_ddi_dns_config_synthesize_address_records_from_https_4be2cd10 field: json.synthesize_address_records_from_https target_field: infoblox_bloxone_ddi.dns_config.synthesize.address_records_from_https if: ctx.json?.synthesize_address_records_from_https != '' @@ -1744,13 +2028,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c059379d field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tags_to_infoblox_bloxone_ddi_dns_config_tags_7daf6b23 field: json.tags target_field: infoblox_bloxone_ddi.dns_config.tags ignore_missing: true - foreach: + tag: foreach_json_transfer_acl_6c97c5d4 field: json.transfer_acl if: ctx.json?.transfer_acl instanceof List processor: @@ -1768,6 +2055,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_transfer_acl_f496b458 field: json.transfer_acl if: ctx.json?.transfer_acl instanceof List processor: @@ -1779,6 +2067,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_transfer_acl_0feaa10b field: json.transfer_acl if: ctx.json?.transfer_acl instanceof List processor: @@ -1790,6 +2079,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_transfer_acl_6a4e7a1c field: json.transfer_acl if: ctx.json?.transfer_acl instanceof List processor: @@ -1800,6 +2090,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_transfer_acl_3976bbcc field: json.transfer_acl if: ctx.json?.transfer_acl instanceof List processor: @@ -1810,10 +2101,12 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_transfer_acl_to_infoblox_bloxone_ddi_dns_config_transfer_acl_5da7a267 field: json.transfer_acl target_field: infoblox_bloxone_ddi.dns_config.transfer_acl ignore_missing: true - foreach: + tag: foreach_json_update_acl_3affea98 field: json.update_acl if: ctx.json?.update_acl instanceof List processor: @@ -1831,6 +2124,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_update_acl_0a2590ec field: json.update_acl if: ctx.json?.update_acl instanceof List processor: @@ -1842,6 +2136,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_update_acl_21d269bf field: json.update_acl if: ctx.json?.update_acl instanceof List processor: @@ -1853,6 +2148,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_update_acl_0640d6b0 field: json.update_acl if: ctx.json?.update_acl instanceof List processor: @@ -1863,6 +2159,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_update_acl_c5ea6868 field: json.update_acl if: ctx.json?.update_acl instanceof List processor: @@ -1873,10 +2170,12 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_json_update_acl_to_infoblox_bloxone_ddi_dns_config_update_acl_9a9768eb field: json.update_acl target_field: infoblox_bloxone_ddi.dns_config.update_acl ignore_missing: true - date: + tag: date_json_updated_at_to_infoblox_bloxone_ddi_dns_config_updated_at_2ed8ed7c field: json.updated_at target_field: infoblox_bloxone_ddi.dns_config.updated_at if: ctx.json?.updated_at != null && ctx.json.updated_at != '' @@ -1884,13 +2183,16 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_d195716d field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_timestamp_513dc060 field: '@timestamp' copy_from: infoblox_bloxone_ddi.dns_config.updated_at ignore_failure: true - convert: + tag: convert_json_use_forwarders_for_subzones_to_infoblox_bloxone_ddi_dns_config_use_forwarders_for_subzones_a316864b field: json.use_forwarders_for_subzones target_field: infoblox_bloxone_ddi.dns_config.use_forwarders_for_subzones if: ctx.json?.use_forwarders_for_subzones != '' @@ -1898,9 +2200,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a27332de field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_zone_authority_default_ttl_to_infoblox_bloxone_ddi_dns_config_zone_authority_default_ttl_f4377f6b field: json.zone_authority.default_ttl target_field: infoblox_bloxone_ddi.dns_config.zone_authority.default_ttl if: ctx.json?.zone_authority?.default_ttl != '' @@ -1908,9 +2212,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_157d2c5c field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_zone_authority_expire_to_infoblox_bloxone_ddi_dns_config_zone_authority_expire_a5398716 field: json.zone_authority.expire target_field: infoblox_bloxone_ddi.dns_config.zone_authority.expire if: ctx.json?.zone_authority?.expire != '' @@ -1918,13 +2224,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c7d9c06f field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_zone_authority_mname_to_infoblox_bloxone_ddi_dns_config_zone_authority_mname_19a61ab3 field: json.zone_authority.mname target_field: infoblox_bloxone_ddi.dns_config.zone_authority.mname ignore_missing: true - convert: + tag: convert_json_zone_authority_negative_ttl_to_infoblox_bloxone_ddi_dns_config_zone_authority_negative_ttl_42b077bf field: json.zone_authority.negative_ttl target_field: infoblox_bloxone_ddi.dns_config.zone_authority.negative_ttl if: ctx.json?.zone_authority?.negative_ttl != '' @@ -1932,17 +2241,21 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_06c1a2fe field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_zone_authority_protocol_mname_to_infoblox_bloxone_ddi_dns_config_zone_authority_protocol_mname_9d6a314e field: json.zone_authority.protocol_mname target_field: infoblox_bloxone_ddi.dns_config.zone_authority.protocol.mname ignore_missing: true - rename: + tag: rename_json_zone_authority_protocol_rname_to_infoblox_bloxone_ddi_dns_config_zone_authority_protocol_rname_49e65ab8 field: json.zone_authority.protocol_rname target_field: infoblox_bloxone_ddi.dns_config.zone_authority.protocol.rname ignore_missing: true - convert: + tag: convert_json_zone_authority_refresh_to_infoblox_bloxone_ddi_dns_config_zone_authority_refresh_b563369a field: json.zone_authority.refresh target_field: infoblox_bloxone_ddi.dns_config.zone_authority.refresh if: ctx.json?.zone_authority?.refresh != '' @@ -1950,9 +2263,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_3fcd6e09 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_zone_authority_retry_to_infoblox_bloxone_ddi_dns_config_zone_authority_retry_65780717 field: json.zone_authority.retry target_field: infoblox_bloxone_ddi.dns_config.zone_authority.retry if: ctx.json?.zone_authority?.retry != '' @@ -1960,13 +2275,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_58138c10 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_zone_authority_rname_to_infoblox_bloxone_ddi_dns_config_zone_authority_rname_4963ee4b field: json.zone_authority.rname target_field: infoblox_bloxone_ddi.dns_config.zone_authority.rname ignore_missing: true - convert: + tag: convert_json_zone_authority_use_default_mname_to_infoblox_bloxone_ddi_dns_config_zone_authority_use_default_mname_f5413b8b field: json.zone_authority.use_default_mname target_field: infoblox_bloxone_ddi.dns_config.zone_authority.use_default_mname if: ctx.json?.zone_authority?.use_default_mname != '' @@ -1974,12 +2292,15 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_cd6a515e field: error.message value: '{{{_ingest.on_failure_message}}}' - remove: + tag: remove_json_29cdffdc field: json ignore_missing: true - remove: + tag: remove_8a5ca587 field: - infoblox_bloxone_ddi.dns_config.updated_at - infoblox_bloxone_ddi.dns_config.lame_ttl @@ -1989,6 +2310,7 @@ processors: ignore_failure: true ignore_missing: true - script: + tag: script_009c1e23 description: Drops null/empty values recursively. lang: painless source: boolean dropEmptyFields(Object object) { if (object == null || object == '') { return true; } else if (object instanceof Map) { ((Map) object).values().removeIf(value -> dropEmptyFields(value)); return (((Map) object).size() == 0); } else if (object instanceof List) { ((List) object).removeIf(value -> dropEmptyFields(value)); return (((List) object).length == 0); } return false; } dropEmptyFields(ctx); diff --git a/packages/infoblox_bloxone_ddi/data_stream/dns_data/elasticsearch/ingest_pipeline/default.yml b/packages/infoblox_bloxone_ddi/data_stream/dns_data/elasticsearch/ingest_pipeline/default.yml index 262b79ec787..15c6896204f 100644 --- a/packages/infoblox_bloxone_ddi/data_stream/dns_data/elasticsearch/ingest_pipeline/default.yml +++ b/packages/infoblox_bloxone_ddi/data_stream/dns_data/elasticsearch/ingest_pipeline/default.yml @@ -2,18 +2,23 @@ description: Pipeline for parsing DNS data logs. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - set: + tag: set_event_type_42379782 field: event.type value: [protocol] - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -25,11 +30,14 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - drop: + tag: drop_88d6232c if: ctx.json?.results instanceof List && ctx.json.results.length == 0 - fingerprint: + tag: fingerprint_8b3026d5 fields: - json.created_at - json.updated_at @@ -37,18 +45,22 @@ processors: target_field: _id ignore_missing: true - rename: + tag: rename_json_absolute_name_spec_to_infoblox_bloxone_ddi_dns_data_absolute_name_spec_c7c3b188 field: json.absolute_name_spec target_field: infoblox_bloxone_ddi.dns_data.absolute_name.spec ignore_missing: true - rename: + tag: rename_json_absolute_zone_name_to_infoblox_bloxone_ddi_dns_data_absolute_zone_name_6c1f5e94 field: json.absolute_zone_name target_field: infoblox_bloxone_ddi.dns_data.absolute_zone.name ignore_missing: true - rename: + tag: rename_json_comment_to_infoblox_bloxone_ddi_dns_data_comment_d572735f field: json.comment target_field: infoblox_bloxone_ddi.dns_data.comment ignore_missing: true - date: + tag: date_json_created_at_to_infoblox_bloxone_ddi_dns_data_created_at_9877ee60 field: json.created_at target_field: infoblox_bloxone_ddi.dns_data.created_at if: ctx.json?.created_at != null && ctx.json.created_at != '' @@ -56,17 +68,21 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_0b05c951 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_event_created_4acb9efd field: event.created copy_from: infoblox_bloxone_ddi.dns_data.created_at ignore_failure: true - rename: + tag: rename_json_delegation_to_infoblox_bloxone_ddi_dns_data_delegation_11353af5 field: json.delegation target_field: infoblox_bloxone_ddi.dns_data.delegation ignore_missing: true - convert: + tag: convert_json_disabled_to_infoblox_bloxone_ddi_dns_data_disabled_ecafe038 field: json.disabled target_field: infoblox_bloxone_ddi.dns_data.disabled if: ctx.json?.disabled != '' @@ -74,45 +90,56 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8b062cf7 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_dns_absolute_name_spec_to_infoblox_bloxone_ddi_dns_data_absolute_name_spec_1c50309b field: json.dns_absolute_name_spec target_field: infoblox_bloxone_ddi.dns_data.absolute.name.spec ignore_missing: true - rename: + tag: rename_json_dns_absolute_zone_name_to_infoblox_bloxone_ddi_dns_data_absolute_zone_name_42e7eec7 field: json.dns_absolute_zone_name target_field: infoblox_bloxone_ddi.dns_data.absolute.zone.name ignore_missing: true - rename: + tag: rename_json_dns_name_in_zone_to_infoblox_bloxone_ddi_dns_data_name_in_zone_63cd06d4 field: json.dns_name_in_zone target_field: infoblox_bloxone_ddi.dns_data.name_in.zone ignore_missing: true - rename: + tag: rename_json_dns_rdata_to_infoblox_bloxone_ddi_dns_data_rdata_value_4b61ff1f field: json.dns_rdata target_field: infoblox_bloxone_ddi.dns_data.rdata_value ignore_missing: true - rename: + tag: rename_json_id_to_infoblox_bloxone_ddi_dns_data_id_d6683119 field: json.id target_field: infoblox_bloxone_ddi.dns_data.id ignore_missing: true - set: + tag: set_event_id_01f980ef field: event.id copy_from: infoblox_bloxone_ddi.dns_data.id ignore_failure: true - rename: + tag: rename_json_inheritance_sources_ttl_action_to_infoblox_bloxone_ddi_dns_data_inheritance_sources_ttl_action_ef6b29b8 field: json.inheritance_sources.ttl.action target_field: infoblox_bloxone_ddi.dns_data.inheritance.sources.ttl.action ignore_missing: true - rename: + tag: rename_json_inheritance_sources_ttl_display_name_to_infoblox_bloxone_ddi_dns_data_inheritance_sources_ttl_display_name_baf8a949 field: json.inheritance_sources.ttl.display_name target_field: infoblox_bloxone_ddi.dns_data.inheritance.sources.ttl.display.name ignore_missing: true - rename: + tag: rename_json_inheritance_sources_ttl_source_to_infoblox_bloxone_ddi_dns_data_inheritance_sources_ttl_source_7c2700e8 field: json.inheritance_sources.ttl.source target_field: infoblox_bloxone_ddi.dns_data.inheritance.sources.ttl.source ignore_missing: true - convert: + tag: convert_json_inheritance_sources_ttl_value_to_infoblox_bloxone_ddi_dns_data_inheritance_sources_ttl_value_01c9543d field: json.inheritance_sources.ttl.value target_field: infoblox_bloxone_ddi.dns_data.inheritance.sources.ttl.value if: ctx.json?.inheritance_sources?.ttl?.value != '' @@ -120,21 +147,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d7b52cbe field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_name_in_zone_to_infoblox_bloxone_ddi_dns_data_name_in_zone_5b5277c5 field: json.name_in_zone target_field: infoblox_bloxone_ddi.dns_data.name_in_zone ignore_missing: true - json: + tag: json_json_options_to_json_options_7bbcb358 field: json.options target_field: json.options if: ctx.json?.options instanceof String on_failure: - append: + tag: append_error_message_64f0f4f7 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_options_create_ptr_to_infoblox_bloxone_ddi_dns_data_options_create_ptr_4698c12c field: json.options.create_ptr target_field: infoblox_bloxone_ddi.dns_data.options.create_ptr if: ctx.json?.options?.create_ptr != '' @@ -142,9 +174,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_66819d01 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_options_check_rmz_to_infoblox_bloxone_ddi_dns_data_options_check_rmz_62172193 field: json.options.check_rmz target_field: infoblox_bloxone_ddi.dns_data.options.check_rmz if: ctx.json?.options?.check_rmz != '' @@ -152,9 +186,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_31d95b9c field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_options_address_to_infoblox_bloxone_ddi_dns_data_options_address_3d1c37a0 field: json.options.address target_field: infoblox_bloxone_ddi.dns_data.options.address if: ctx.json?.options?.address != '' @@ -162,18 +198,22 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_08a2d6c9 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_related_ip_ff94030f field: related.ip value: '{{{infoblox_bloxone_ddi.dns_data.options.address}}}' allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_provider_metadata_to_infoblox_bloxone_ddi_dns_data_provider_metadata_97dc4243 field: json.provider_metadata target_field: infoblox_bloxone_ddi.dns_data.provider_metadata ignore_missing: true - convert: + tag: convert_json_rdata_address_to_infoblox_bloxone_ddi_dns_data_rdata_address_cef682e8 field: json.rdata.address target_field: infoblox_bloxone_ddi.dns_data.rdata.address if: ctx.json?.rdata?.address != '' @@ -181,42 +221,52 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_33d5b3d5 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_related_ip_cf143be7 field: related.ip value: '{{{infoblox_bloxone_ddi.dns_data.rdata.address}}}' allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_rdata_flags_to_infoblox_bloxone_ddi_dns_data_rdata_flags_17ff8acf field: json.rdata.flags target_field: infoblox_bloxone_ddi.dns_data.rdata.flags ignore_missing: true - rename: + tag: rename_json_rdata_tag_to_infoblox_bloxone_ddi_dns_data_rdata_tag_14fbe2f9 field: json.rdata.tag target_field: infoblox_bloxone_ddi.dns_data.rdata.tag ignore_missing: true - rename: + tag: rename_json_rdata_value_to_infoblox_bloxone_ddi_dns_data_rdata_value_8588396b field: json.rdata.value target_field: infoblox_bloxone_ddi.dns_data.rdata.value ignore_missing: true - rename: + tag: rename_json_rdata_cname_to_infoblox_bloxone_ddi_dns_data_rdata_cname_0e0787fd field: json.rdata.cname target_field: infoblox_bloxone_ddi.dns_data.rdata.cname ignore_missing: true - rename: + tag: rename_json_rdata_target_to_infoblox_bloxone_ddi_dns_data_rdata_target_a93d33a1 field: json.rdata.target target_field: infoblox_bloxone_ddi.dns_data.rdata.target ignore_missing: true - rename: + tag: rename_json_rdata_dhcid_to_infoblox_bloxone_ddi_dns_data_rdata_dhcid_e5746351 field: json.rdata.dhcid target_field: infoblox_bloxone_ddi.dns_data.rdata.dhcid ignore_missing: true - rename: + tag: rename_json_rdata_exchange_to_infoblox_bloxone_ddi_dns_data_rdata_exchange_447d66dd field: json.rdata.exchange target_field: infoblox_bloxone_ddi.dns_data.rdata.exchange ignore_missing: true - convert: + tag: convert_json_rdata_preference_to_infoblox_bloxone_ddi_dns_data_rdata_preference_b01e05da field: json.rdata.preference target_field: infoblox_bloxone_ddi.dns_data.rdata.preference if: ctx.json?.rdata?.preference != '' @@ -224,9 +274,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_b8bd16c9 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_rdata_order_to_infoblox_bloxone_ddi_dns_data_rdata_order_5f8d80d9 field: json.rdata.order target_field: infoblox_bloxone_ddi.dns_data.rdata.order if: ctx.json?.rdata?.order != '' @@ -234,25 +286,31 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5c168750 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_rdata_regexp_to_infoblox_bloxone_ddi_dns_data_rdata_regexp_c587cc2d field: json.rdata.regexp target_field: infoblox_bloxone_ddi.dns_data.rdata.regexp ignore_missing: true - rename: + tag: rename_json_rdata_replacement_to_infoblox_bloxone_ddi_dns_data_rdata_replacement_9d6a6551 field: json.rdata.replacement target_field: infoblox_bloxone_ddi.dns_data.rdata.replacement ignore_missing: true - rename: + tag: rename_json_rdata_services_to_infoblox_bloxone_ddi_dns_data_rdata_services_55dbe8c5 field: json.rdata.services target_field: infoblox_bloxone_ddi.dns_data.rdata.services ignore_missing: true - rename: + tag: rename_json_rdata_dname_to_infoblox_bloxone_ddi_dns_data_rdata_dname_57395dd7 field: json.rdata.dname target_field: infoblox_bloxone_ddi.dns_data.rdata.dname ignore_missing: true - convert: + tag: convert_json_rdata_expire_to_infoblox_bloxone_ddi_dns_data_rdata_expire_b5b46e44 field: json.rdata.expire target_field: infoblox_bloxone_ddi.dns_data.rdata.expire if: ctx.json?.rdata?.expire != '' @@ -260,13 +318,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ed02f9e7 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_rdata_mname_to_infoblox_bloxone_ddi_dns_data_rdata_mname_b86c2101 field: json.rdata.mname target_field: infoblox_bloxone_ddi.dns_data.rdata.mname ignore_missing: true - convert: + tag: convert_json_rdata_negative_ttl_to_infoblox_bloxone_ddi_dns_data_rdata_negative_ttl_50254b8b field: json.rdata.negative_ttl target_field: infoblox_bloxone_ddi.dns_data.rdata.negative_ttl if: ctx.json?.rdata?.negative_ttl != '' @@ -274,9 +335,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f327d578 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_rdata_refresh_to_infoblox_bloxone_ddi_dns_data_rdata_refresh_c635c204 field: json.rdata.refresh target_field: infoblox_bloxone_ddi.dns_data.rdata.refresh if: ctx.json?.rdata?.refresh != '' @@ -284,9 +347,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_20866515 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_rdata_retry_to_infoblox_bloxone_ddi_dns_data_rdata_retry_cef00b23 field: json.rdata.retry target_field: infoblox_bloxone_ddi.dns_data.rdata.retry if: ctx.json?.rdata?.retry != '' @@ -294,13 +359,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_03114eca field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_rdata_rname_to_infoblox_bloxone_ddi_dns_data_rdata_rname_5581b567 field: json.rdata.rname target_field: infoblox_bloxone_ddi.dns_data.rdata.rname ignore_missing: true - convert: + tag: convert_json_rdata_serial_to_infoblox_bloxone_ddi_dns_data_rdata_serial_4e1e7f7b field: json.rdata.serial target_field: infoblox_bloxone_ddi.dns_data.rdata.serial if: ctx.json?.rdata?.serial != '' @@ -308,9 +376,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_513bcca8 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_rdata_port_to_infoblox_bloxone_ddi_dns_data_rdata_port_7fdd932c field: json.rdata.port target_field: infoblox_bloxone_ddi.dns_data.rdata.port if: ctx.json?.rdata?.port != '' @@ -318,9 +388,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_b4d4632b field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_rdata_priority_to_infoblox_bloxone_ddi_dns_data_rdata_priority_53dc424b field: json.rdata.priority target_field: infoblox_bloxone_ddi.dns_data.rdata.priority if: ctx.json?.rdata?.priority != '' @@ -328,9 +400,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_968e8cc8 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_rdata_weight_to_infoblox_bloxone_ddi_dns_data_rdata_weight_3cc4931b field: json.rdata.weight target_field: infoblox_bloxone_ddi.dns_data.rdata.weight if: ctx.json?.rdata?.weight != '' @@ -338,17 +412,21 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_fdfb34bc field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_rdata_text_to_infoblox_bloxone_ddi_dns_data_rdata_text_a393df8d field: json.rdata.text target_field: infoblox_bloxone_ddi.dns_data.rdata.text ignore_missing: true - rename: + tag: rename_json_rdata_type_to_infoblox_bloxone_ddi_dns_data_rdata_type_df32d01d field: json.rdata.type target_field: infoblox_bloxone_ddi.dns_data.rdata.type ignore_missing: true - convert: + tag: convert_json_rdata_length_kind_to_infoblox_bloxone_ddi_dns_data_rdata_length_kind_b06e8558 field: json.rdata.length_kind target_field: infoblox_bloxone_ddi.dns_data.rdata.length_kind if: ctx.json?.rdata?.length_kind != '' @@ -356,17 +434,21 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e3e8e899 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tags_to_infoblox_bloxone_ddi_dns_data_tags_ebb43a95 field: json.tags target_field: infoblox_bloxone_ddi.dns_data.tags ignore_missing: true - rename: + tag: rename_json_source_to_infoblox_bloxone_ddi_dns_data_source_fabb7ca1 field: json.source target_field: infoblox_bloxone_ddi.dns_data.source ignore_missing: true - convert: + tag: convert_json_ttl_to_infoblox_bloxone_ddi_dns_data_ttl_ab9d1224 field: json.ttl target_field: infoblox_bloxone_ddi.dns_data.ttl if: ctx.json?.ttl != '' @@ -374,13 +456,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_01f5f2bf field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_type_to_infoblox_bloxone_ddi_dns_data_type_b357b9b5 field: json.type target_field: infoblox_bloxone_ddi.dns_data.type ignore_missing: true - date: + tag: date_json_updated_at_to_infoblox_bloxone_ddi_dns_data_updated_at_d8710eaa field: json.updated_at target_field: infoblox_bloxone_ddi.dns_data.updated_at if: ctx.json?.updated_at != null && ctx.json.updated_at != '' @@ -388,37 +473,46 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_89128807 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_timestamp_e9b4d9b6 field: '@timestamp' copy_from: infoblox_bloxone_ddi.dns_data.updated_at ignore_failure: true - rename: + tag: rename_json_view_to_infoblox_bloxone_ddi_dns_data_view_b3c9af95 field: json.view target_field: infoblox_bloxone_ddi.dns_data.view ignore_missing: true - rename: + tag: rename_json_view_name_to_infoblox_bloxone_ddi_dns_data_view_name_fcd81eff field: json.view_name target_field: infoblox_bloxone_ddi.dns_data.view_name ignore_missing: true - rename: + tag: rename_json_zone_to_infoblox_bloxone_ddi_dns_data_zone_57c2f2e5 field: json.zone target_field: infoblox_bloxone_ddi.dns_data.zone ignore_missing: true - set: + tag: set_dns_answers_data_dd97ca2a field: dns.answers.data copy_from: infoblox_bloxone_ddi.dns_data.rdata_value ignore_empty_value: true - set: + tag: set_dns_answers_ttl_a5fe837e field: dns.answers.ttl copy_from: infoblox_bloxone_ddi.dns_data.ttl ignore_empty_value: true - set: + tag: set_dns_answers_type_571cabca field: dns.answers.type copy_from: infoblox_bloxone_ddi.dns_data.type ignore_empty_value: true - script: + tag: script_24a2b4bd lang: painless if: ctx.dns?.answers != null source: | @@ -426,25 +520,31 @@ processors: a.add(ctx.dns.answers); ctx.dns.answers = a; - set: + tag: set_dns_question_type_525b0bef field: dns.question.type copy_from: infoblox_bloxone_ddi.dns_data.type ignore_empty_value: true - set: + tag: set_dns_question_name_8e26b957 field: dns.question.name copy_from: infoblox_bloxone_ddi.dns_data.absolute.name.spec ignore_empty_value: true - set: + tag: set_dns_question_registered_domain_a93b90d2 field: dns.question.registered_domain copy_from: infoblox_bloxone_ddi.dns_data.absolute.zone.name ignore_empty_value: true - set: + tag: set_dns_question_subdomain_5a197dce field: dns.question.subdomain copy_from: infoblox_bloxone_ddi.dns_data.name_in.zone ignore_empty_value: true - remove: + tag: remove_json_29cdffdc field: json ignore_missing: true - remove: + tag: remove_06aa9cba field: - infoblox_bloxone_ddi.dns_data.updated_at - infoblox_bloxone_ddi.dns_data.lame_ttl @@ -459,6 +559,7 @@ processors: ignore_failure: true ignore_missing: true - script: + tag: script_e2ccbdd7 description: Drops null/empty values recursively. lang: painless source: | diff --git a/packages/infoblox_bloxone_ddi/manifest.yml b/packages/infoblox_bloxone_ddi/manifest.yml index a4a19e920df..1cc60e556a1 100644 --- a/packages/infoblox_bloxone_ddi/manifest.yml +++ b/packages/infoblox_bloxone_ddi/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.3.2" name: infoblox_bloxone_ddi title: Infoblox BloxOne DDI -version: "1.23.1" +version: "1.23.2" description: Collect logs from Infoblox BloxOne DDI with Elastic Agent. type: integration categories: diff --git a/packages/island_browser/changelog.yml b/packages/island_browser/changelog.yml index e4d04dd2158..f3a5d59e493 100644 --- a/packages/island_browser/changelog.yml +++ b/packages/island_browser/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.1.4" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.1.3" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/island_browser/data_stream/admin_actions/elasticsearch/ingest_pipeline/default.yml b/packages/island_browser/data_stream/admin_actions/elasticsearch/ingest_pipeline/default.yml index c604e3e63df..0eeb9115341 100644 --- a/packages/island_browser/data_stream/admin_actions/elasticsearch/ingest_pipeline/default.yml +++ b/packages/island_browser/data_stream/admin_actions/elasticsearch/ingest_pipeline/default.yml @@ -81,6 +81,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_928fbf8f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -92,6 +93,7 @@ processors: if: ctx.json?.createdDate != null && ctx.json.createdDate != '' on_failure: - append: + tag: append_error_message_6b6a3e3d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -187,6 +189,7 @@ processors: if: ctx.json?.sourceIp != '' on_failure: - append: + tag: append_error_message_7ba57a17 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -229,6 +232,7 @@ processors: if: ctx.json?.timestamp != null && ctx.json.timestamp != '' on_failure: - append: + tag: append_error_message_3550c002 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -245,6 +249,7 @@ processors: if: ctx.json?.updatedDate != null && ctx.json.updatedDate != '' on_failure: - append: + tag: append_error_message_da016b32 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -314,6 +319,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/island_browser/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/island_browser/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index e82ddbd9050..4bc220d5faa 100644 --- a/packages/island_browser/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/island_browser/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -69,6 +69,7 @@ processors: if: ctx.json?.createdDate != null && ctx.json.createdDate != '' on_failure: - append: + tag: append_error_message_ca78b705 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -83,6 +84,7 @@ processors: target_field: island_browser.audit.details on_failure: - append: + tag: append_error_message_b752380b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -161,6 +163,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9ae22d55 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -171,6 +174,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_bbaddbfa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -255,6 +259,7 @@ processors: if: ctx.json?.processedDate != null && ctx.json.processedDate != '' on_failure: - append: + tag: append_error_message_729566ad field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -266,6 +271,7 @@ processors: if: ctx.json?.publicIp != '' on_failure: - append: + tag: append_error_message_8aefec8b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -274,6 +280,7 @@ processors: copy_from: island_browser.audit.public_ip ignore_empty_value: true - geoip: + tag: geoip_source_nat_ip_to_source_geo_8f47b04f field: source.nat.ip target_field: source.geo ignore_missing: true @@ -352,6 +359,7 @@ processors: if: ctx.json?.sourceIp != '' on_failure: - append: + tag: append_error_message_b7b4148b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -395,6 +403,7 @@ processors: if: ctx.json?.timestamp != null && ctx.json.timestamp != '' on_failure: - append: + tag: append_error_message_3691d87a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -418,6 +427,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8a1a3eb8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -434,6 +444,7 @@ processors: if: ctx.json?.updatedDate != null && ctx.json.updatedDate != '' on_failure: - append: + tag: append_error_message_eb29aeda field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -449,6 +460,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ebab7cdb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -565,6 +577,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/island_browser/data_stream/compromised_credential/elasticsearch/ingest_pipeline/default.yml b/packages/island_browser/data_stream/compromised_credential/elasticsearch/ingest_pipeline/default.yml index 2308d413882..5ea42f801df 100644 --- a/packages/island_browser/data_stream/compromised_credential/elasticsearch/ingest_pipeline/default.yml +++ b/packages/island_browser/data_stream/compromised_credential/elasticsearch/ingest_pipeline/default.yml @@ -47,6 +47,7 @@ processors: if: ctx.json?.compromisedDate != null && ctx.json.compromisedDate != '' on_failure: - append: + tag: append_error_message_242f66d0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -58,6 +59,7 @@ processors: if: ctx.json?.createdDate != null && ctx.json.createdDate != '' on_failure: - append: + tag: append_error_message_68f13462 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -130,6 +132,7 @@ processors: if: ctx.json?.updatedDate != null && ctx.json.updatedDate != '' on_failure: - append: + tag: append_error_message_89bdae91 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -197,6 +200,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/island_browser/data_stream/device/elasticsearch/ingest_pipeline/default.yml b/packages/island_browser/data_stream/device/elasticsearch/ingest_pipeline/default.yml index f2cba7ea45f..012c9b31766 100644 --- a/packages/island_browser/data_stream/device/elasticsearch/ingest_pipeline/default.yml +++ b/packages/island_browser/data_stream/device/elasticsearch/ingest_pipeline/default.yml @@ -117,6 +117,7 @@ processors: if: ctx.json?.createdDate != null && ctx.json.createdDate != '' on_failure: - append: + tag: append_error_message_759b6d9e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -142,6 +143,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_525a5156 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -162,6 +164,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_6ac72788 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -204,9 +207,11 @@ processors: if: ctx.json?.externalIpAddress != '' on_failure: - append: + tag: append_error_message_fd3efd5f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - geoip: + tag: geoip_island_browser_device_external_ip_address_to_host_geo_c0801aed field: island_browser.device.external_ip_address target_field: host.geo ignore_missing: true @@ -286,6 +291,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_7fcb9002 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -321,6 +327,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f7afbf53 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -331,6 +338,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_08d0495a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -341,6 +349,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_47e01b97 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -351,6 +360,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4736aaa4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -361,6 +371,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_37f0e328 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -371,6 +382,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d7501d6a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -381,6 +393,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_25762401 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -391,6 +404,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5517745e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -422,6 +436,7 @@ processors: if: ctx.json?.lastSeen != null && ctx.json.lastSeen != '' on_failure: - append: + tag: append_error_message_aeab9339 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -541,6 +556,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_20e85f74 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -551,6 +567,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_98c7b102 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -591,6 +608,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_af7a9287 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -628,6 +646,7 @@ processors: } on_failure: - append: + tag: append_error_message_b7ef02dd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -638,6 +657,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_db59ae4b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -670,6 +690,7 @@ processors: if: ctx.json?.policyUpdateTime != null && ctx.json.policyUpdateTime != '' on_failure: - append: + tag: append_error_message_69c9ca3b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -680,6 +701,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_429d3d3a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -690,6 +712,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_552ecc60 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -710,6 +733,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_7f075333 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -725,6 +749,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9c6c41a4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -735,6 +760,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_049ec87f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -745,6 +771,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_83a5e00f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -766,6 +793,7 @@ processors: if: ctx.json?.updatedDate != null && ctx.json.updatedDate != '' on_failure: - append: + tag: append_error_message_5634fed5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -866,6 +894,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/island_browser/data_stream/user/elasticsearch/ingest_pipeline/default.yml b/packages/island_browser/data_stream/user/elasticsearch/ingest_pipeline/default.yml index 6d0e4b4c113..2001319cd2f 100644 --- a/packages/island_browser/data_stream/user/elasticsearch/ingest_pipeline/default.yml +++ b/packages/island_browser/data_stream/user/elasticsearch/ingest_pipeline/default.yml @@ -62,6 +62,7 @@ processors: if: ctx.json?.createdDate != null && ctx.json.createdDate != '' on_failure: - append: + tag: append_error_message_8075909b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -98,6 +99,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_929e6300 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -109,6 +111,7 @@ processors: if: ctx.json?.expirationDate != null && ctx.json.expirationDate != '' on_failure: - append: + tag: append_error_message_be2d5e36 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -127,6 +130,7 @@ processors: target_field: island_browser.user.last_name ignore_missing: true - set: + tag: set_user_full_name_bcf195b1 field: user.full_name value: '{{{island_browser.user.first_name}}} {{{island_browser.user.last_name}}}' if: ctx.island_browser?.user?.first_name != null && ctx.island_browser?.user?.last_name != null @@ -165,6 +169,7 @@ processors: if: ctx.json?.invitationDate != null && ctx.json.invitationDate != '' on_failure: - append: + tag: append_error_message_82885250 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -176,6 +181,7 @@ processors: if: ctx.json?.lastLogin != null && ctx.json.lastLogin != '' on_failure: - append: + tag: append_error_message_0bda148a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -187,6 +193,7 @@ processors: if: ctx.json?.lastSeen != null && ctx.json.lastSeen != '' on_failure: - append: + tag: append_error_message_9d63b972 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -213,6 +220,7 @@ processors: if: ctx.json?.updatedDate != null && ctx.json.updatedDate != '' on_failure: - append: + tag: append_error_message_c83ef5f0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -296,6 +304,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/island_browser/manifest.yml b/packages/island_browser/manifest.yml index bc8f489b5e9..3c46cf76a27 100644 --- a/packages/island_browser/manifest.yml +++ b/packages/island_browser/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: island_browser title: Island Browser -version: 1.1.3 +version: 1.1.4 description: Collect logs from Island Browser with Elastic Agent. type: integration categories: diff --git a/packages/jamf_compliance_reporter/changelog.yml b/packages/jamf_compliance_reporter/changelog.yml index f5e1edfe7ff..41d5368edd3 100644 --- a/packages/jamf_compliance_reporter/changelog.yml +++ b/packages/jamf_compliance_reporter/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.16.1" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.16.0" changes: - description: Update Kibana constraint to support 9.0.0. diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/default.yml index fae7fdbe302..e442973c675 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -2,30 +2,38 @@ description: Pipeline for Jamf Compliance Reporter logs. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to_json_cac66847 field: event.original target_field: json ignore_failure: true - pipeline: + tag: pipeline_ae75b026 name: '{{ IngestPipeline "pipeline_app_metrics" }}' if: ctx.json?.header?.event_name == 'APP_METRICS' - pipeline: + tag: pipeline_0affc98c name: '{{ IngestPipeline "pipeline_audit" }}' if: ctx.json?.header?.event_name?.startsWith('AUE_') == true - pipeline: + tag: pipeline_e6a6abf5 name: '{{ IngestPipeline "pipeline_event" }}' if: "['AUDIO_VIDEO_DEVICE_EVENT','AUDIT_CLASS_VERIFICATION_EVENT','COMPLIANCE_REPORTER_TAMPER_EVENT','FILE_EVENT','GATEKEEPER_INFO_EVENT','GATEKEEPER_MANUAL_OVERRIDES','GATEKEEPER_QUARANTINE_LOG','HARDWARE_EVENT','LICENSE_INFO_EVENT','PREFERENCE_LIST_EVENT','PRINT_EVENT_INFORMATION','PROHIBITED_APP_BLOCKED','SIGNAL_EVENT','UNIFIED_LOG_EVENT','XPROTECT_DEFINITIONS_VERSION_INFO','XPROTECT_EVENT_LOG'].contains(ctx.json?.header?.event_name)" - remove: + tag: remove_json_29cdffdc field: json ignore_missing: true - script: + tag: script_e2ccbdd7 description: Drops null/empty values recursively. lang: painless source: | diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_app_metrics.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_app_metrics.yml index 59a240b8272..632820681df 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_app_metrics.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_app_metrics.yml @@ -2,136 +2,168 @@ description: Pipeline for Jamf Compliance Reporter app metrics logs. processors: - append: + tag: append_event_type_8a66ccaa field: event.type value: info - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_jamf_compliance_reporter_log_dataset_39bd98fd field: jamf_compliance_reporter.log.dataset value: app_metrics - set: + tag: set_host_os_type_074dd92b field: host.os.type value: macos - append: + tag: append_event_category_8a525c81 field: event.category value: process - convert: + tag: convert_json__event_score_to_jamf_compliance_reporter_log_event_score_3cb39b47 field: json._event_score type: long target_field: jamf_compliance_reporter.log.event_score ignore_missing: true on_failure: - append: + tag: append_error_message_58844c12 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_app_metric_info_cpu_percentage_to_jamf_compliance_reporter_log_app_metric_info_cpu_percentage_e8ed8cd3 field: json.app_metric_info.cpu_percentage type: double target_field: jamf_compliance_reporter.log.app_metric_info.cpu_percentage ignore_missing: true on_failure: - append: + tag: append_error_message_69d5e638 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_app_metric_info_cpu_time_seconds_to_jamf_compliance_reporter_log_app_metric_info_cpu_time_seconds_1e684117 field: json.app_metric_info.cpu_time_seconds type: double target_field: jamf_compliance_reporter.log.app_metric_info.cpu_time_seconds ignore_missing: true on_failure: - append: + tag: append_error_message_32f3cf14 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_app_metric_info_interrupt_wakeups_to_jamf_compliance_reporter_log_app_metric_info_interrupt_wakeups_9645f820 field: json.app_metric_info.interrupt_wakeups type: long target_field: jamf_compliance_reporter.log.app_metric_info.interrupt_wakeups ignore_missing: true on_failure: - append: + tag: append_error_message_9d44dceb field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_app_metric_info_platform_idle_wakeups_to_jamf_compliance_reporter_log_app_metric_info_platform_idle_wakeups_f2f94096 field: json.app_metric_info.platform_idle_wakeups type: long target_field: jamf_compliance_reporter.log.app_metric_info.platform_idle_wakeups ignore_missing: true on_failure: - append: + tag: append_error_message_dd2c42a9 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_app_metric_info_resident_memory_size_mb_to_jamf_compliance_reporter_log_app_metric_info_resident_memory_size_mb_fe345f00 field: json.app_metric_info.resident_memory_size_mb type: double target_field: jamf_compliance_reporter.log.app_metric_info.resident_memory_size.mb ignore_missing: true on_failure: - append: + tag: append_error_message_dcdbf45f field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_app_metric_info_virtual_memory_size_mb_to_jamf_compliance_reporter_log_app_metric_info_virtual_memory_size_mb_f9084642 field: json.app_metric_info.virtual_memory_size_mb type: double target_field: jamf_compliance_reporter.log.app_metric_info.virtual_memory_size.mb ignore_missing: true on_failure: - append: + tag: append_error_message_84554e61 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_header_event_name_to_event_action_99289c09 field: json.header.event_name target_field: event.action ignore_missing: true - lowercase: + tag: lowercase_event_action_9334b869 field: event.action ignore_missing: true - date: + tag: date_json_header_time_seconds_epoch_726cfcbc field: json.header.time_seconds_epoch if: ctx.json?.header?.time_seconds_epoch != 0 formats: - UNIX on_failure: - append: + tag: append_error_message_3be7c1d1 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_host_info_host_name_to_host_hostname_b155f2aa field: json.host_info.host_name target_field: host.hostname ignore_missing: true - append: + tag: append_related_hosts_e6462550 field: related.hosts value: '{{{host.hostname}}}' if: ctx.host?.hostname != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_host_info_host_uuid_to_jamf_compliance_reporter_log_host_info_host_uuid_73fc6247 field: json.host_info.host_uuid target_field: jamf_compliance_reporter.log.host_info.host.uuid ignore_missing: true - rename: + tag: rename_json_host_info_osversion_to_host_os_version_d3d10e97 field: json.host_info.osversion target_field: host.os.version ignore_missing: true - append: + tag: append_host_mac_ddef4d66 field: host.mac value: '{{{json.host_info.primary_mac_address}}}' if: ctx.json?.host_info?.primary_mac_address != null allow_duplicates: false ignore_failure: true - gsub: + tag: gsub_host_mac_d1faae6a field: host.mac pattern: '[-:.]' replacement: '-' ignore_missing: true - uppercase: + tag: uppercase_host_mac_73cd4413 field: host.mac ignore_missing: true - rename: + tag: rename_json_host_info_serial_number_to_host_id_02030592 field: json.host_info.serial_number target_field: host.id ignore_missing: true - script: + tag: script_d350855a lang: painless if: ctx.json?.app_metric_info?.cpu_percentage != null source: | @@ -144,3 +176,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_audio_video_device_event.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_audio_video_device_event.yml index ea529fa52f7..963e26cd8c1 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_audio_video_device_event.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_audio_video_device_event.yml @@ -2,39 +2,47 @@ description: Pipeline for Jamf Compliance Reporter audio video device event logs. processors: - rename: + tag: rename_json_audio_video_device_info_audio_device_creator_to_jamf_compliance_reporter_log_audio_video_device_info_audio_device_creator_7723b83b field: json.audio_video_device_info.audio_device_creator target_field: jamf_compliance_reporter.log.audio_video_device_info.audio_device.creator ignore_missing: true - convert: + tag: convert_json_audio_video_device_info_audio_device_hog_mode_to_jamf_compliance_reporter_log_audio_video_device_info_audio_device_hog_mode_e6c2c650 field: json.audio_video_device_info.audio_device_hog_mode target_field: jamf_compliance_reporter.log.audio_video_device_info.audio_device.hog_mode type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_audio_video_device_info_audio_device_id_to_jamf_compliance_reporter_log_audio_video_device_info_audio_device_id_f1572ca4 field: json.audio_video_device_info.audio_device_id target_field: jamf_compliance_reporter.log.audio_video_device_info.audio_device.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_audio_video_device_info_audio_device_manufacturer_to_jamf_compliance_reporter_log_audio_video_device_info_audio_device_manufacturer_c2a67e77 field: json.audio_video_device_info.audio_device_manufacturer target_field: jamf_compliance_reporter.log.audio_video_device_info.audio_device.manufacturer ignore_missing: true - convert: + tag: convert_json_audio_video_device_info_audio_device_running_to_jamf_compliance_reporter_log_audio_video_device_info_audio_device_running_acb6af65 field: json.audio_video_device_info.audio_device_running target_field: jamf_compliance_reporter.log.audio_video_device_info.audio_device.running type: long ignore_missing: true on_failure: - append: + tag: append_error_message_7489ac52 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_audio_video_device_info_audio_device_uuid_to_jamf_compliance_reporter_log_audio_video_device_info_audio_device_uuid_2322bf13 field: json.audio_video_device_info.audio_device_uuid target_field: jamf_compliance_reporter.log.audio_video_device_info.audio_device.uuid ignore_missing: true - rename: + tag: rename_json_audio_video_device_info_device_status_to_jamf_compliance_reporter_log_audio_video_device_info_device_status_5ff9ae3a field: json.audio_video_device_info.device_status target_field: jamf_compliance_reporter.log.audio_video_device_info.device_status ignore_missing: true @@ -45,3 +53,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_audit.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_audit.yml index ad2ef387bfe..e3d0449e9b4 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_audit.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_audit.yml @@ -2,37 +2,45 @@ description: Pipeline for Jamf Compliance Reporter audit logs. processors: - set: + tag: set_jamf_compliance_reporter_log_dataset_e01ccb3b field: jamf_compliance_reporter.log.dataset value: audit - convert: + tag: convert_json__event_score_to_jamf_compliance_reporter_log_event_score_3cb39b47 field: json._event_score target_field: jamf_compliance_reporter.log.event_score type: long ignore_missing: true on_failure: - append: + tag: append_error_message_58844c12 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_header_event_id_to_event_code_ea6a00b9 field: json.header.event_id target_field: event.code type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_header_event_modifier_to_jamf_compliance_reporter_log_header_event_modifier_fc1dc809 field: json.header.event_modifier target_field: jamf_compliance_reporter.log.header.event_modifier type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_header_event_name_to_event_action_99289c09 field: json.header.event_name target_field: event.action ignore_missing: true - lowercase: + tag: lowercase_event_action_9334b869 field: event.action ignore_missing: true - script: + tag: script_cf20eab3 lang: painless if: ctx.json?.header?.time_seconds_epoch != null && ctx.json.header.time_seconds_epoch != 0 source: | @@ -41,323 +49,399 @@ processors: ctx.json.time_milliseconds = ctx.json.time_milliseconds + (long)ctx.json.header.time_milliseconds_offset; } - date: + tag: date_json_time_milliseconds_55ec1666 field: json.time_milliseconds if: ctx.json?.time_milliseconds != null && ctx.json.time_milliseconds != 0 formats: - UNIX_MS on_failure: - append: + tag: append_error_message_10ae8f9f field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_header_version_to_jamf_compliance_reporter_log_header_version_41b3cab3 field: json.header.version target_field: jamf_compliance_reporter.log.header.version type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_host_info_host_name_to_host_hostname_b155f2aa field: json.host_info.host_name target_field: host.hostname ignore_missing: true - append: + tag: append_related_hosts_e6462550 field: related.hosts value: '{{{host.hostname}}}' if: ctx.host?.hostname != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_host_info_host_uuid_to_jamf_compliance_reporter_log_host_info_host_uuid_73fc6247 field: json.host_info.host_uuid target_field: jamf_compliance_reporter.log.host_info.host.uuid ignore_missing: true - rename: + tag: rename_json_host_info_osversion_to_host_os_version_d3d10e97 field: json.host_info.osversion target_field: host.os.version ignore_missing: true - append: + tag: append_host_mac_ddef4d66 field: host.mac value: '{{{json.host_info.primary_mac_address}}}' if: ctx.json?.host_info?.primary_mac_address != null allow_duplicates: false ignore_failure: true - gsub: + tag: gsub_host_mac_d1faae6a field: host.mac pattern: '[-:.]' replacement: '-' ignore_missing: true - uppercase: + tag: uppercase_host_mac_73cd4413 field: host.mac ignore_missing: true - rename: + tag: rename_json_host_info_serial_number_to_host_id_02030592 field: json.host_info.serial_number target_field: host.id ignore_missing: true - rename: + tag: rename_json_return_description_to_jamf_compliance_reporter_log_return_description_1f80b478 field: json.return.description target_field: jamf_compliance_reporter.log.return.description ignore_missing: true - convert: + tag: convert_json_return_error_to_error_code_4de20ffe field: json.return.error target_field: error.code type: string ignore_missing: true ignore_failure: true - set: + tag: set_event_outcome_d9c276d7 field: event.outcome value: 'success' if: ctx.error?.code == '0' - set: + tag: set_event_outcome_6b78e138 field: event.outcome value: 'failure' if: ctx.error?.code != '0' - convert: + tag: convert_json_return_return_value_to_process_exit_code_a60d08a9 field: json.return.return_value target_field: process.exit_code type: long ignore_missing: true on_failure: - append: + tag: append_error_message_754f0640 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_subject_audit_id_to_process_real_user_id_25b75fe6 field: json.subject.audit_id target_field: process.real_user.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_subject_audit_user_name_to_process_real_user_name_9efe64b1 field: json.subject.audit_user_name target_field: process.real_user.name ignore_missing: true - append: + tag: append_related_user_54bf3eff field: related.user value: '{{{process.real_user.name}}}' if: ctx.process?.real_user?.name != null allow_duplicates: false ignore_failure: true - append: + tag: append_user_name_6d699a67 field: user.name value: '{{{json.subject.audit_user_name}}}' if: ctx.json?.subject?.audit_user_name != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_subject_effective_group_id_to_jamf_compliance_reporter_log_subject_effective_group_id_08ee1ee7 field: json.subject.effective_group_id target_field: jamf_compliance_reporter.log.subject.effective.group.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_subject_effective_group_name_to_jamf_compliance_reporter_log_subject_effective_group_name_c257af46 field: json.subject.effective_group_name target_field: jamf_compliance_reporter.log.subject.effective.group.name ignore_missing: true - convert: + tag: convert_json_subject_effective_user_id_to_process_user_id_4f74864d field: json.subject.effective_user_id target_field: process.user.id type: string ignore_missing: true ignore_failure: true - append: + tag: append_user_name_6426d56f field: user.name value: '{{{json.subject.effective_user_name}}}' if: ctx.json?.subject?.effective_user_name != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_6dba669f field: related.user value: '{{{json.subject.effective_user_name}}}' if: ctx.json?.subject?.effective_user_name != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_subject_effective_user_name_to_process_user_name_5691197e field: json.subject.effective_user_name target_field: process.user.name ignore_missing: true - convert: + tag: convert_json_subject_group_id_to_process_real_group_id_ea90ca9c field: json.subject.group_id target_field: process.real_group.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_subject_group_name_to_process_real_group_name_81890bc5 field: json.subject.group_name target_field: process.real_group.name ignore_missing: true - rename: + tag: rename_json_subject_process_hash_to_process_hash_sha1_ec183692 field: json.subject.process_hash target_field: process.hash.sha1 ignore_missing: true - append: + tag: append_related_hash_8db6a16a field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_subject_process_id_to_jamf_compliance_reporter_log_subject_process_pid_c6218d3b field: json.subject.process_id target_field: jamf_compliance_reporter.log.subject.process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_b970a4a6 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_subject_process_name_to_jamf_compliance_reporter_log_subject_process_name_4a7b4c83 field: json.subject.process_name target_field: jamf_compliance_reporter.log.subject.process.name ignore_missing: true - convert: + tag: convert_json_subject_session_id_to_jamf_compliance_reporter_log_subject_session_id_f51f0574 field: json.subject.session_id target_field: jamf_compliance_reporter.log.subject.session.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_subject_terminal_id_addr_to_jamf_compliance_reporter_log_subject_terminal_id_addr_44248987 field: json.subject.terminal_id.addr target_field: jamf_compliance_reporter.log.subject.terminal_id.addr type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_subject_terminal_id_ip_address_9dc8b10a field: json.subject.terminal_id.ip_address type: ip ignore_missing: true on_failure: - remove: + tag: remove_json_subject_terminal_id_ip_address_d1da3917 field: json.subject.terminal_id.ip_address ignore_missing: true - append: + tag: append_error_message_4c3b7861 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_host_ip_e4f236dd field: host.ip value: '{{{json.subject.terminal_id.ip_address}}}' if: ctx.json?.subject?.terminal_id?.ip_address != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_ip_8541d760 field: related.ip value: '{{{json.subject.terminal_id.ip_address}}}' if: ctx.json?.subject?.terminal_id?.ip_address != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_subject_terminal_id_port_to_jamf_compliance_reporter_log_subject_terminal_id_port_301d7d36 field: json.subject.terminal_id.port target_field: jamf_compliance_reporter.log.subject.terminal_id.port type: long ignore_missing: true on_failure: - append: + tag: append_error_message_9d728ff1 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_subject_terminal_id_type_to_jamf_compliance_reporter_log_subject_terminal_id_type_8a893ed5 field: json.subject.terminal_id.type target_field: jamf_compliance_reporter.log.subject.terminal_id.type type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_subject_user_id_to_user_id_19eb600e field: json.subject.user_id target_field: user.id type: string ignore_missing: true ignore_failure: true - append: + tag: append_user_name_11035ae3 field: user.name value: '{{{json.subject.user_name}}}' if: ctx.json?.subject?.user_name != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_acadb613 field: related.user value: '{{{json.subject.user_name}}}' if: ctx.json?.subject?.user_name != null allow_duplicates: false ignore_failure: true - append: + tag: append_event_type_8a66ccaa field: event.type value: info - set: + tag: set_event_kind_de80643c field: event.kind value: event - append: + tag: append_event_category_109b4d72 field: event.category value: authentication - pipeline: + tag: pipeline_e67931fd name: '{{ IngestPipeline "pipeline_aue_accept" }}' if: ctx.event.action == 'aue_accept' - pipeline: + tag: pipeline_db24270d name: '{{ IngestPipeline "pipeline_aue_auth" }}' if: '["aue_auth_user", "aue_ssauthorize", "aue_ssauthmech"].contains(ctx.event.action)' - pipeline: + tag: pipeline_01455bfc name: '{{ IngestPipeline "pipeline_aue_bind_and_aue_connect" }}' if: '["aue_bind", "aue_connect"].contains(ctx.event.action)' - pipeline: + tag: pipeline_29a8ddcb name: '{{ IngestPipeline "pipeline_aue_chdir" }}' if: ctx.event.action == 'aue_chdir' - pipeline: + tag: pipeline_42230e85 name: '{{ IngestPipeline "pipeline_aue_chroot" }}' if: ctx.event.action == 'aue_chroot' - pipeline: + tag: pipeline_32b565c5 name: '{{ IngestPipeline "pipeline_aue_execve" }}' if: ctx.event.action == 'aue_execve' - pipeline: + tag: pipeline_09900d59 name: '{{ IngestPipeline "pipeline_aue_exit" }}' if: ctx.event.action == 'aue_exit' - pipeline: + tag: pipeline_2a284975 name: '{{ IngestPipeline "pipeline_aue_kill" }}' if: ctx.event.action == 'aue_kill' - pipeline: + tag: pipeline_65aa7105 name: '{{ IngestPipeline "pipeline_aue_mount" }}' if: ctx.event.action == 'aue_mount' - pipeline: + tag: pipeline_40e4c839 name: '{{ IngestPipeline "pipeline_aue_posix_spawn" }}' if: ctx.event.action == 'aue_posix_spawn' - pipeline: + tag: pipeline_12d162b4 name: '{{ IngestPipeline "pipeline_aue_remove_from_group_and_aue_mac_set_proc" }}' if: '["aue_remove_from_group", "aue_mac_set_proc"].contains(ctx.event.action)' - pipeline: + tag: pipeline_b5be0f75 name: '{{ IngestPipeline "pipeline_aue_session" }}' if: '["aue_session_end", "aue_session_update", "aue_session_close", "aue_session_start"].contains(ctx.event.action)' - pipeline: + tag: pipeline_2885bfec name: '{{ IngestPipeline "pipeline_aue_arguments" }}' if: '["aue_setsockopt", "aue_shutdown"].contains(ctx.event.action)' - pipeline: + tag: pipeline_f505d005 name: '{{ IngestPipeline "pipeline_aue_ssauthint" }}' if: ctx.event.action == 'aue_ssauthint' - pipeline: + tag: pipeline_473cfe75 name: '{{ IngestPipeline "pipeline_aue_tasknameforpid" }}' if: ctx.event.action == 'aue_tasknameforpid' - pipeline: + tag: pipeline_635d4cb3 name: '{{ IngestPipeline "pipeline_aue_unmount" }}' if: ctx.event.action == 'aue_unmount' - pipeline: + tag: pipeline_160e3cc1 name: '{{ IngestPipeline "pipeline_aue_fork" }}' if: ctx.event.action == 'aue_fork' - pipeline: + tag: pipeline_a2dab2bf name: '{{ IngestPipeline "pipeline_identity_object" }}' if: '["aue_getauid", "aue_lw_login", "aue_settimeofday"].contains(ctx.event.action)' - pipeline: + tag: pipeline_5df0f67d name: '{{ IngestPipeline "pipeline_aue_listen" }}' if: ctx.event.action == 'aue_listen' - pipeline: + tag: pipeline_27fc8ba5 name: '{{ IngestPipeline "pipeline_aue_logout" }}' if: ctx.event.action == 'aue_logout' - pipeline: + tag: pipeline_9810aa85 name: '{{ IngestPipeline "pipeline_aue_pidfortask" }}' if: ctx.event.action == 'aue_pidfortask' - pipeline: + tag: pipeline_c6af489d name: '{{ IngestPipeline "pipeline_aue_ptrace" }}' if: ctx.event.action == 'aue_ptrace' - pipeline: + tag: pipeline_8c52fdaf name: '{{ IngestPipeline "pipeline_aue_setpriority" }}' if: ctx.event.action == 'aue_setpriority' - pipeline: + tag: pipeline_e23326b5 name: '{{ IngestPipeline "pipeline_aue_socketpair" }}' if: ctx.event.action == 'aue_socketpair' - pipeline: + tag: pipeline_8018a705 name: '{{ IngestPipeline "pipeline_aue_taskforpid" }}' if: ctx.event.action == 'aue_taskforpid' - pipeline: + tag: pipeline_daa3f04d name: '{{ IngestPipeline "pipeline_aue_wait4" }}' if: ctx.event.action == 'aue_wait4' on_failure: @@ -367,3 +451,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_audit_class_verification_event.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_audit_class_verification_event.yml index 4b3d62a7800..2f6c9c197db 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_audit_class_verification_event.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_audit_class_verification_event.yml @@ -2,29 +2,35 @@ description: Pipeline for Jamf Compliance Reporter audit class verification event logs. processors: - rename: + tag: rename_json_audit_class_verification_info_contents_to_jamf_compliance_reporter_log_audit_class_verification_info_contents_72d5edba field: json.audit_class_verification_info.contents target_field: jamf_compliance_reporter.log.audit_class_verification_info.contents ignore_missing: true - rename: + tag: rename_json_audit_class_verification_info_osversion_to_jamf_compliance_reporter_log_audit_class_verification_info_os_version_bed69b0c field: json.audit_class_verification_info.osversion target_field: jamf_compliance_reporter.log.audit_class_verification_info.os.version ignore_missing: true - convert: + tag: convert_json_audit_class_verification_info_restored_default_to_jamf_compliance_reporter_log_audit_class_verification_info_restored_default_664e7cd0 field: json.audit_class_verification_info.restored_default target_field: jamf_compliance_reporter.log.audit_class_verification_info.restored_default type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_19b98c19 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_audit_class_verification_info_status_to_jamf_compliance_reporter_log_audit_class_verification_info_status_527dc7eb field: json.audit_class_verification_info.status target_field: jamf_compliance_reporter.log.audit_class_verification_info.status type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_audit_class_verification_info_status_str_to_jamf_compliance_reporter_log_audit_class_verification_info_status_str_9d29f242 field: json.audit_class_verification_info.status_str target_field: jamf_compliance_reporter.log.audit_class_verification_info.status_str ignore_missing: true @@ -35,3 +41,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_accept.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_accept.yml index 6c5c4484a56..7b75a1d34eb 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_accept.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_accept.yml @@ -2,26 +2,31 @@ description: Pipeline for Jamf Compliance Reporter aue_accept audit logs. processors: - rename: + tag: rename_json_path_to_jamf_compliance_reporter_log_path_dcaf10e2 field: json.path target_field: jamf_compliance_reporter.log.path ignore_missing: true - convert: + tag: convert_json_socket_unix_family_to_json_inet_family_a3c3ec3c field: json.socket_unix.family target_field: json.inet_family type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_socket_unix_path_to_jamf_compliance_reporter_log_socket_unix_path_2702e3a9 field: json.socket_unix.path target_field: jamf_compliance_reporter.log.socket.unix.path ignore_missing: true - convert: + tag: convert_json_arguments_fd_to_jamf_compliance_reporter_log_arguments_fd_5367d19d field: json.arguments.fd target_field: jamf_compliance_reporter.log.arguments.fd type: string ignore_missing: true ignore_failure: true - script: + tag: script_bf7d3e4d description: Dynamically map Socket Address Families. lang: painless if: ctx.json?.inet_family != null @@ -57,6 +62,7 @@ processors: map.put('25', 'pseudo_AF_PIP'); ctx.jamf_compliance_reporter.log.socket.unix.family = map.get(ctx.json.inet_family); - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' on_failure: - set: @@ -65,3 +71,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_arguments.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_arguments.yml index 22ae0015ef4..a3a7743e66b 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_arguments.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_arguments.yml @@ -2,12 +2,14 @@ description: Pipeline for Jamf Compliance Reporter aue_arguments audit logs. processors: - convert: + tag: convert_json_arguments_fd_to_jamf_compliance_reporter_log_arguments_fd_5367d19d field: json.arguments.fd target_field: jamf_compliance_reporter.log.arguments.fd type: string ignore_missing: true ignore_failure: true - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' on_failure: - set: @@ -16,3 +18,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_auth.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_auth.yml index 88f31fe2f91..4c850887260 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_auth.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_auth.yml @@ -2,8 +2,10 @@ description: Pipeline for Jamf Compliance Reporter aue_auth audit logs. processors: - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' - rename: + tag: rename_json_texts_to_jamf_compliance_reporter_log_texts_e75677b2 field: json.texts target_field: jamf_compliance_reporter.log.texts ignore_missing: true @@ -14,3 +16,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_bind_and_aue_connect.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_bind_and_aue_connect.yml index 7736882d466..e5715bcc48d 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_bind_and_aue_connect.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_bind_and_aue_connect.yml @@ -2,57 +2,68 @@ description: Pipeline for Jamf Compliance Reporter aue_bind and aue_connect audit logs. processors: - convert: + tag: convert_json_socket_inet_addr_to_jamf_compliance_reporter_log_socket_inet_addr_073bdb7c field: json.socket_inet.addr target_field: jamf_compliance_reporter.log.socket.inet.addr type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_arguments_fd_to_jamf_compliance_reporter_log_arguments_fd_5367d19d field: json.arguments.fd target_field: jamf_compliance_reporter.log.arguments.fd type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_socket_inet_family_to_json_inet_family_da3c0fb0 field: json.socket_inet.family target_field: json.inet_family type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_socket_inet_id_to_jamf_compliance_reporter_log_socket_inet_id_1d335ca8 field: json.socket_inet.id target_field: jamf_compliance_reporter.log.socket.inet.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_socket_inet_ip_address_to_server_ip_b5e6ed72 field: json.socket_inet.ip_address target_field: server.ip type: ip ignore_missing: true on_failure: - remove: + tag: remove_json_socket_inet_ip_address_f04d008f field: json.socket_inet.ip_address ignore_missing: true - append: + tag: append_error_message_106415f7 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_related_ip_3ed00a18 field: related.ip value: '{{{server.ip}}}' if: ctx.server?.ip != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_socket_inet_port_to_server_port_91beb58f field: json.socket_inet.port target_field: server.port type: long ignore_missing: true on_failure: - append: + tag: append_error_message_6c7bb514 field: error.message value: '{{{_ingest.on_failure_message}}}' - script: + tag: script_4643c989 description: Dynamically map Socket Address Families. lang: painless if: ctx.json?.inet_family != null @@ -88,6 +99,7 @@ processors: map.put('25', 'pseudo_AF_PIP'); ctx.jamf_compliance_reporter.log.socket.inet.family = map.get(ctx.json.inet_family); - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' on_failure: - set: @@ -96,3 +108,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_chdir.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_chdir.yml index 331a9c99661..fdeb5be57c9 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_chdir.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_chdir.yml @@ -2,67 +2,80 @@ description: Pipeline for Jamf Compliance Reporter aue_chdir audit logs. processors: - convert: + tag: convert_json_attributes_device_to_jamf_compliance_reporter_log_attributes_device_d89a67d9 field: json.attributes.device target_field: jamf_compliance_reporter.log.attributes.device type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_attributes_file_access_mode_to_json_file_access_mode_3515c381 field: json.attributes.file_access_mode target_field: json.file_access_mode ignore_failure: true - convert: + tag: convert_json_attributes_file_system_id_to_jamf_compliance_reporter_log_attributes_file_system_id_cc879847 field: json.attributes.file_system_id target_field: jamf_compliance_reporter.log.attributes.file.system.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_attributes_node_id_to_jamf_compliance_reporter_log_attributes_node_id_38b0771e field: json.attributes.node_id target_field: jamf_compliance_reporter.log.attributes.node.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_attributes_owner_group_id_to_user_group_id_f8a03798 field: json.attributes.owner_group_id target_field: user.group.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_attributes_owner_group_name_to_user_group_name_834f090f field: json.attributes.owner_group_name target_field: user.group.name ignore_missing: true - convert: + tag: convert_json_attributes_owner_user_id_58bbc519 field: json.attributes.owner_user_id type: string ignore_missing: true ignore_failure: true - append: + tag: append_user_id_f10ea64d field: user.id value: '{{{json.attributes.owner_user_id}}}' if: ctx.json?.attributes?.owner_user_id != null allow_duplicates: false ignore_failure: true - append: + tag: append_user_name_2ba08467 field: user.name value: '{{{json.attributes.owner_user_name}}}' if: ctx.json?.attributes?.owner_user_name != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_0321d5d7 field: related.user value: '{{{json.attributes.owner_user_name}}}' if: ctx.json?.attributes?.owner_user_name != null allow_duplicates: false ignore_failure: true - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' - rename: + tag: rename_json_path_to_jamf_compliance_reporter_log_path_dcaf10e2 field: json.path target_field: jamf_compliance_reporter.log.path ignore_missing: true - script: + tag: script_fbc9f1e5 description: Convert Decimal into Octal. lang: painless if: ctx.json?.file_access_mode != null @@ -76,3 +89,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_chroot.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_chroot.yml index baab69174ee..4da117cdb90 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_chroot.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_chroot.yml @@ -2,69 +2,83 @@ description: Pipeline for Jamf Compliance Reporter aue_chroot audit logs. processors: - convert: + tag: convert_json_attributes_device_to_jamf_compliance_reporter_log_attributes_device_d89a67d9 field: json.attributes.device target_field: jamf_compliance_reporter.log.attributes.device type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_attributes_file_access_mode_to_json_file_access_mode_58473c4f field: json.attributes.file_access_mode target_field: json.file_access_mode ignore_missing: true - convert: + tag: convert_json_attributes_file_system_id_to_jamf_compliance_reporter_log_attributes_file_system_id_cc879847 field: json.attributes.file_system_id target_field: jamf_compliance_reporter.log.attributes.file.system.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_attributes_node_id_to_jamf_compliance_reporter_log_attributes_node_id_38b0771e field: json.attributes.node_id target_field: jamf_compliance_reporter.log.attributes.node.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_attributes_owner_group_id_to_user_group_id_f8a03798 field: json.attributes.owner_group_id target_field: user.group.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_attributes_owner_group_name_to_user_group_name_834f090f field: json.attributes.owner_group_name target_field: user.group.name ignore_missing: true - convert: + tag: convert_json_attributes_owner_user_id_58bbc519 field: json.attributes.owner_user_id type: string ignore_missing: true ignore_failure: true - append: + tag: append_user_id_f10ea64d field: user.id value: '{{{json.attributes.owner_user_id}}}' if: ctx.json?.attributes?.owner_user_id != null allow_duplicates: false ignore_failure: true - append: + tag: append_user_name_2ba08467 field: user.name value: '{{{json.attributes.owner_user_name}}}' if: ctx.json?.attributes?.owner_user_name != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_0321d5d7 field: related.user value: '{{{json.attributes.owner_user_name}}}' if: ctx.json?.attributes?.owner_user_name != null allow_duplicates: false ignore_failure: true - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' - rename: + tag: rename_json_path_to_jamf_compliance_reporter_log_path_dcaf10e2 field: json.path target_field: jamf_compliance_reporter.log.path ignore_missing: true - pipeline: + tag: pipeline_75de012d name: '{{ IngestPipeline "pipeline_exec_chain_child_object" }}' - script: + tag: script_fbc9f1e5 description: Convert Decimal into Octal. lang: painless if: ctx.json?.file_access_mode != null @@ -78,3 +92,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_execve.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_execve.yml index 997c3bf5bd7..75a58a0e40a 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_execve.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_execve.yml @@ -2,77 +2,93 @@ description: Pipeline for Jamf Compliance Reporter aue_execve audit logs. processors: - convert: + tag: convert_json_attributes_device_to_jamf_compliance_reporter_log_attributes_device_d89a67d9 field: json.attributes.device target_field: jamf_compliance_reporter.log.attributes.device type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_attributes_file_access_mode_to_json_file_access_mode_3515c381 field: json.attributes.file_access_mode target_field: json.file_access_mode ignore_failure: true - convert: + tag: convert_json_attributes_file_system_id_to_jamf_compliance_reporter_log_attributes_file_system_id_cc879847 field: json.attributes.file_system_id target_field: jamf_compliance_reporter.log.attributes.file.system.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_attributes_node_id_to_jamf_compliance_reporter_log_attributes_node_id_38b0771e field: json.attributes.node_id target_field: jamf_compliance_reporter.log.attributes.node.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_attributes_owner_group_id_to_user_group_id_f8a03798 field: json.attributes.owner_group_id target_field: user.group.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_attributes_owner_group_name_to_user_group_name_834f090f field: json.attributes.owner_group_name target_field: user.group.name ignore_missing: true - convert: + tag: convert_json_attributes_owner_user_id_58bbc519 field: json.attributes.owner_user_id type: string ignore_missing: true ignore_failure: true - append: + tag: append_user_id_f10ea64d field: user.id value: '{{{json.attributes.owner_user_id}}}' if: ctx.json?.attributes?.owner_user_id != null allow_duplicates: false ignore_failure: true - append: + tag: append_user_name_2ba08467 field: user.name value: '{{{json.attributes.owner_user_name}}}' if: ctx.json?.attributes?.owner_user_name != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_0321d5d7 field: related.user value: '{{{json.attributes.owner_user_name}}}' if: ctx.json?.attributes?.owner_user_name != null allow_duplicates: false ignore_failure: true - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' - rename: + tag: rename_json_path_to_jamf_compliance_reporter_log_path_dcaf10e2 field: json.path target_field: jamf_compliance_reporter.log.path ignore_missing: true - pipeline: + tag: pipeline_75de012d name: '{{ IngestPipeline "pipeline_exec_chain_child_object" }}' - rename: + tag: rename_json_exec_args_args_to_json_args_c2752467 field: json.exec_args.args target_field: json.args ignore_missing: true - rename: + tag: rename_json_exec_args_args_compiled_to_jamf_compliance_reporter_log_exec_args_args_compiled_6e9208ee field: json.exec_args.args_compiled target_field: jamf_compliance_reporter.log.exec_args.args_compiled ignore_missing: true - script: + tag: script_fdd491a2 lang: painless if: ctx.json?.exec_env?.env?.ARCH != null && ctx.json.exec_env.env.ARCH != '' params: @@ -105,30 +121,37 @@ processors: ctx.host.os.put('type', ctx.json.exec_env.env.ARCH); ctx.json.exec_env.env.remove('ARCH'); - rename: + tag: rename_json_exec_env_env_CPU_to_host_architecture_a6731bc7 field: json.exec_env.env.CPU target_field: host.architecture ignore_missing: true - rename: + tag: rename_json_exec_env_env_MALWAREBYTES_GROUP_to_jamf_compliance_reporter_log_exec_env_env_malwarebytes_group_bc0c67ca field: json.exec_env.env.MALWAREBYTES_GROUP target_field: jamf_compliance_reporter.log.exec_env.env.malwarebytes_group ignore_missing: true - rename: + tag: rename_json_exec_env_env_PATH_to_jamf_compliance_reporter_log_exec_env_env_path_66b7d43e field: json.exec_env.env.PATH target_field: jamf_compliance_reporter.log.exec_env.env.path ignore_missing: true - rename: + tag: rename_json_exec_env_env_XPC_FLAGS_to_jamf_compliance_reporter_log_exec_env_env_xpc_flags_10e3a485 field: json.exec_env.env.XPC_FLAGS target_field: jamf_compliance_reporter.log.exec_env.env.xpc.flags ignore_missing: true - rename: + tag: rename_json_exec_env_env_XPC_SERVICE_NAME_to_jamf_compliance_reporter_log_exec_env_env_xpc_service_name_b71ea807 field: json.exec_env.env.XPC_SERVICE_NAME target_field: jamf_compliance_reporter.log.exec_env.env.xpc.service_name ignore_missing: true - rename: + tag: rename_json_exec_env_env_compiled_to_jamf_compliance_reporter_log_exec_env_env_compiled_c264f32b field: json.exec_env.env_compiled target_field: jamf_compliance_reporter.log.exec_env.env.compiled ignore_missing: true - script: + tag: script_721d76b6 description: Convert Object into Array. lang: painless source: | @@ -140,6 +163,7 @@ processors: } } - script: + tag: script_fbc9f1e5 description: Convert Decimal into Octal. lang: painless if: ctx.json?.file_access_mode != null @@ -153,3 +177,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_exit.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_exit.yml index 9e5d229b6e6..c9c99258b64 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_exit.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_exit.yml @@ -2,19 +2,24 @@ description: Pipeline for Jamf Compliance Reporter aue_exit audit logs. processors: - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' - pipeline: + tag: pipeline_75de012d name: '{{ IngestPipeline "pipeline_exec_chain_child_object" }}' - convert: + tag: convert_json_exit_return_value_to_jamf_compliance_reporter_log_exit_return_value_80eec135 field: json.exit.return_value target_field: jamf_compliance_reporter.log.exit.return.value type: long ignore_missing: true on_failure: - append: + tag: append_error_message_3d22b642 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_exit_status_to_jamf_compliance_reporter_log_exit_status_b34443dd field: json.exit.status target_field: jamf_compliance_reporter.log.exit.status type: string @@ -27,3 +32,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_fork.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_fork.yml index e71ae41fc8c..743008f4816 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_fork.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_fork.yml @@ -2,19 +2,23 @@ description: Pipeline for Jamf Compliance Reporter aue_fork audit logs. processors: - rename: + tag: rename_json_exec_chain_parent_uuid_to_jamf_compliance_reporter_log_exec_chain_parent_uuid_2d473082 field: json.exec_chain_parent.uuid target_field: jamf_compliance_reporter.log.exec_chain_parent.uuid ignore_missing: true - convert: + tag: convert_json_arguments_child_PID_to_jamf_compliance_reporter_log_arguments_child_pid_bd346fed field: json.arguments.child_PID target_field: jamf_compliance_reporter.log.arguments.child.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_be26f75e field: error.message value: '{{{_ingest.on_failure_message}}}' - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' on_failure: - set: @@ -23,3 +27,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_kill.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_kill.yml index 27de8efdfc9..d93df24e3d2 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_kill.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_kill.yml @@ -2,14 +2,17 @@ description: Pipeline for Jamf Compliance Reporter aue_kill audit logs. processors: - convert: + tag: convert_json_arguments_signal_to_jamf_compliance_reporter_log_arguments_signal_e16e6e31 field: json.arguments.signal target_field: jamf_compliance_reporter.log.arguments.signal type: string ignore_missing: true ignore_failure: true - pipeline: + tag: pipeline_b038445a name: '{{ IngestPipeline "pipeline_process_object" }}' - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' on_failure: - set: @@ -18,3 +21,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_listen.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_listen.yml index bf2b0986bbf..6f1eab24147 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_listen.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_listen.yml @@ -2,14 +2,17 @@ description: Pipeline for Jamf Compliance Reporter aue_listen audit logs. processors: - convert: + tag: convert_json_arguments_fd_to_jamf_compliance_reporter_log_arguments_fd_5367d19d field: json.arguments.fd target_field: jamf_compliance_reporter.log.arguments.fd type: string ignore_missing: true ignore_failure: true - pipeline: + tag: pipeline_75de012d name: '{{ IngestPipeline "pipeline_exec_chain_child_object" }}' - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' on_failure: - set: @@ -18,3 +21,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_logout.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_logout.yml index 3c01e5fd382..e44d20c1c1c 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_logout.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_logout.yml @@ -2,8 +2,10 @@ description: Pipeline for Jamf Compliance Reporter aue_logout audit logs. processors: - pipeline: + tag: pipeline_75de012d name: '{{ IngestPipeline "pipeline_exec_chain_child_object" }}' - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' on_failure: - set: @@ -12,3 +14,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_mount.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_mount.yml index 15817ff08dd..f810db989dc 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_mount.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_mount.yml @@ -2,79 +2,95 @@ description: Pipeline for Jamf Compliance Reporter aue_mount audit logs. processors: - rename: + tag: rename_json_texts_to_jamf_compliance_reporter_log_texts_e75677b2 field: json.texts target_field: jamf_compliance_reporter.log.texts ignore_missing: true - pipeline: + tag: pipeline_75de012d name: '{{ IngestPipeline "pipeline_exec_chain_child_object" }}' - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' - rename: + tag: rename_json_path_to_jamf_compliance_reporter_log_path_dcaf10e2 field: json.path target_field: jamf_compliance_reporter.log.path ignore_missing: true - convert: + tag: convert_json_attributes_device_to_jamf_compliance_reporter_log_attributes_device_d89a67d9 field: json.attributes.device target_field: jamf_compliance_reporter.log.attributes.device type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_attributes_file_access_mode_to_json_file_access_mode_58473c4f field: json.attributes.file_access_mode target_field: json.file_access_mode ignore_missing: true - convert: + tag: convert_json_attributes_file_system_id_to_jamf_compliance_reporter_log_attributes_file_system_id_cc879847 field: json.attributes.file_system_id target_field: jamf_compliance_reporter.log.attributes.file.system.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_attributes_node_id_to_jamf_compliance_reporter_log_attributes_node_id_38b0771e field: json.attributes.node_id target_field: jamf_compliance_reporter.log.attributes.node.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_attributes_owner_group_id_to_jamf_compliance_reporter_log_attributes_owner_group_id_0c9a22e5 field: json.attributes.owner_group_id target_field: jamf_compliance_reporter.log.attributes.owner.group.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_attributes_owner_group_name_to_jamf_compliance_reporter_log_attributes_owner_group_name_580265ea field: json.attributes.owner_group_name target_field: jamf_compliance_reporter.log.attributes.owner.group.name ignore_missing: true - convert: + tag: convert_json_attributes_owner_user_id_58bbc519 field: json.attributes.owner_user_id type: string ignore_missing: true ignore_failure: true - append: + tag: append_user_id_f10ea64d field: user.id value: '{{{json.attributes.owner_user_id}}}' if: ctx.json?.attributes?.owner_user_id != null allow_duplicates: false ignore_failure: true - append: + tag: append_user_name_2ba08467 field: user.name value: '{{{json.attributes.owner_user_name}}}' if: ctx.json?.attributes?.owner_user_name != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_0321d5d7 field: related.user value: '{{{json.attributes.owner_user_name}}}' if: ctx.json?.attributes?.owner_user_name != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_arguments_flags_to_jamf_compliance_reporter_log_arguments_flags_bd432fc5 field: json.arguments.flags target_field: jamf_compliance_reporter.log.arguments.flags type: string ignore_missing: true ignore_failure: true - script: + tag: script_20002992 description: Convert Decimal into Octal. lang: painless if: ctx.json?.file_access_mode != null @@ -88,3 +104,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_pidfortask.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_pidfortask.yml index 47e70323f13..e566e3a858b 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_pidfortask.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_pidfortask.yml @@ -2,21 +2,25 @@ description: Pipeline for Jamf Compliance Reporter aue_pidfortask audit logs. processors: - convert: + tag: convert_json_arguments_pid_to_jamf_compliance_reporter_log_arguments_pid_2d5a7310 field: json.arguments.pid target_field: jamf_compliance_reporter.log.arguments.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_7eecdb63 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_arguments_port_to_jamf_compliance_reporter_log_arguments_port_05cb9806 field: json.arguments.port target_field: jamf_compliance_reporter.log.arguments.port type: long ignore_missing: true on_failure: - append: + tag: append_error_message_a6148881 field: error.message value: '{{{_ingest.on_failure_message}}}' on_failure: @@ -26,3 +30,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_posix_spawn.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_posix_spawn.yml index 0817db223fa..833c1f19bb7 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_posix_spawn.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_posix_spawn.yml @@ -2,39 +2,48 @@ description: Pipeline for Jamf Compliance Reporter aue_posix_spawn audit logs. processors: - convert: + tag: convert_json_arguments_child_PID_to_jamf_compliance_reporter_log_arguments_child_pid_bd346fed field: json.arguments.child_PID target_field: jamf_compliance_reporter.log.arguments.child.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_be26f75e field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_exec_args_args_to_json_args_c2752467 field: json.exec_args.args target_field: json.args ignore_missing: true - rename: + tag: rename_json_exec_args_args_compiled_to_jamf_compliance_reporter_log_exec_args_args_compiled_6e9208ee field: json.exec_args.args_compiled target_field: jamf_compliance_reporter.log.exec_args.args_compiled ignore_missing: true - rename: + tag: rename_json_exec_env_env_XPC_FLAGS_to_jamf_compliance_reporter_log_exec_env_env_xpc_flags_10e3a485 field: json.exec_env.env.XPC_FLAGS target_field: jamf_compliance_reporter.log.exec_env.env.xpc.flags ignore_missing: true - rename: + tag: rename_json_exec_env_env_compiled_to_jamf_compliance_reporter_log_exec_env_env_compiled_c264f32b field: json.exec_env.env_compiled target_field: jamf_compliance_reporter.log.exec_env.env.compiled ignore_missing: true - rename: + tag: rename_json_path_to_jamf_compliance_reporter_log_path_dcaf10e2 field: json.path target_field: jamf_compliance_reporter.log.path ignore_missing: true - rename: + tag: rename_json_exec_chain_parent_uuid_to_jamf_compliance_reporter_log_exec_chain_parent_uuid_2d473082 field: json.exec_chain_parent.uuid target_field: jamf_compliance_reporter.log.exec_chain_parent.uuid ignore_missing: true - script: + tag: script_721d76b6 description: Convert Object into Array. lang: painless source: | @@ -46,6 +55,7 @@ processors: } } - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' on_failure: - set: @@ -54,3 +64,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_ptrace.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_ptrace.yml index d18ab79acc1..741e77a8e2d 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_ptrace.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_ptrace.yml @@ -2,32 +2,38 @@ description: Pipeline for Jamf Compliance Reporter aue_ptrace audit logs. processors: - convert: + tag: convert_json_arguments_addr_to_jamf_compliance_reporter_log_arguments_addr_7ab0f7f3 field: json.arguments.addr target_field: jamf_compliance_reporter.log.arguments.addr type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_arguments_data_to_jamf_compliance_reporter_log_arguments_data_4c8f8b05 field: json.arguments.data target_field: jamf_compliance_reporter.log.arguments.data type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_arguments_process_to_jamf_compliance_reporter_log_arguments_process_ee3dbde9 field: json.arguments.process target_field: jamf_compliance_reporter.log.arguments.process type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_arguments_request_to_jamf_compliance_reporter_log_arguments_request_e96cea55 field: json.arguments.request target_field: jamf_compliance_reporter.log.arguments.request type: string ignore_missing: true ignore_failure: true - pipeline: + tag: pipeline_75de012d name: '{{ IngestPipeline "pipeline_exec_chain_child_object" }}' - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' on_failure: - set: @@ -36,3 +42,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_remove_from_group_and_aue_mac_set_proc.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_remove_from_group_and_aue_mac_set_proc.yml index cd2cb021cef..9b72570d117 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_remove_from_group_and_aue_mac_set_proc.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_remove_from_group_and_aue_mac_set_proc.yml @@ -2,12 +2,15 @@ description: Pipeline for Jamf Compliance Reporter aue_remove_from_group and aue_mac_set_proc audit logs. processors: - rename: + tag: rename_json_texts_to_jamf_compliance_reporter_log_texts_e75677b2 field: json.texts target_field: jamf_compliance_reporter.log.texts ignore_missing: true - pipeline: + tag: pipeline_75de012d name: '{{ IngestPipeline "pipeline_exec_chain_child_object" }}' - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' on_failure: - set: @@ -16,3 +19,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_session.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_session.yml index dc13e5e80bd..f8401aa90a2 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_session.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_session.yml @@ -2,18 +2,21 @@ description: Pipeline for Jamf Compliance Reporter aue_session_start, aue_session_end, aue_session_update and aue_session_close audit logs. processors: - convert: + tag: convert_json_arguments_am_failure_to_jamf_compliance_reporter_log_arguments_am_failure_9070e193 field: json.arguments.am_failure target_field: jamf_compliance_reporter.log.arguments.am_failure type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_arguments_am_success_to_jamf_compliance_reporter_log_arguments_am_success_2aaf72fd field: json.arguments.am_success target_field: jamf_compliance_reporter.log.arguments.am_success type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_arguments_sflags_to_jamf_compliance_reporter_log_arguments_sflags_5ced041d field: json.arguments.sflags target_field: jamf_compliance_reporter.log.arguments.sflags type: string @@ -26,3 +29,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_setpriority.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_setpriority.yml index 1506b89b819..a097e802abd 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_setpriority.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_setpriority.yml @@ -2,23 +2,28 @@ description: Pipeline for Jamf Compliance Reporter aue_setpriority audit logs. processors: - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' - convert: + tag: convert_json_arguments_priority_to_jamf_compliance_reporter_log_arguments_priority_1076537e field: json.arguments.priority target_field: jamf_compliance_reporter.log.arguments.priority type: long ignore_missing: true on_failure: - append: + tag: append_error_message_99c855b1 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_arguments_which_to_jamf_compliance_reporter_log_arguments_which_4be357b5 field: json.arguments.which target_field: jamf_compliance_reporter.log.arguments.which type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_arguments_who_to_jamf_compliance_reporter_log_arguments_who_d04fef91 field: json.arguments.who target_field: jamf_compliance_reporter.log.arguments.who type: string @@ -31,3 +36,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_socketpair.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_socketpair.yml index 3b484d41361..f5be301546d 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_socketpair.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_socketpair.yml @@ -2,20 +2,24 @@ description: Pipeline for Jamf Compliance Reporter aue_socketpair audit logs. processors: - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' - convert: + tag: convert_json_arguments_domain_to_jamf_compliance_reporter_log_arguments_domain_f9220ed5 field: json.arguments.domain target_field: jamf_compliance_reporter.log.arguments.domain type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_arguments_protocol_to_jamf_compliance_reporter_log_arguments_protocol_142f1611 field: json.arguments.protocol target_field: jamf_compliance_reporter.log.arguments.protocol type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_arguments_type_to_jamf_compliance_reporter_log_arguments_type_ecaf0ac5 field: json.arguments.type target_field: jamf_compliance_reporter.log.arguments.type type: string @@ -28,3 +32,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_ssauthint.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_ssauthint.yml index c7a9f04e7f7..6a3250d657f 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_ssauthint.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_ssauthint.yml @@ -2,18 +2,22 @@ description: Pipeline for Jamf Compliance Reporter aue_ssauthint audit logs. processors: - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' - rename: + tag: rename_json_texts_to_jamf_compliance_reporter_log_texts_e75677b2 field: json.texts target_field: jamf_compliance_reporter.log.texts ignore_missing: true - convert: + tag: convert_json_arguments_known_UID__to_jamf_compliance_reporter_log_arguments_known_uid_726a3bb6 field: json.arguments.known_UID_ target_field: jamf_compliance_reporter.log.arguments.known_uid type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_arguments_to_jamf_compliance_reporter_log_arguments_flattened_58f7e5e1 field: json.arguments target_field: jamf_compliance_reporter.log.arguments.flattened ignore_missing: true @@ -24,3 +28,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_taskforpid.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_taskforpid.yml index 4b80cdc52bc..40493002f52 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_taskforpid.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_taskforpid.yml @@ -2,24 +2,29 @@ description: Pipeline for Jamf Compliance Reporter aue_taskforpid audit logs. processors: - convert: + tag: convert_json_arguments_target_port_to_jamf_compliance_reporter_log_arguments_target_port_cc0dcd91 field: json.arguments.target_port target_field: jamf_compliance_reporter.log.arguments.target.port type: long ignore_missing: true on_failure: - append: + tag: append_error_message_64d1e576 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_arguments_task_port_to_jamf_compliance_reporter_log_arguments_task_port_2fe271c9 field: json.arguments.task_port target_field: jamf_compliance_reporter.log.arguments.task.port type: long ignore_missing: true on_failure: - append: + tag: append_error_message_ddba9a56 field: error.message value: '{{{_ingest.on_failure_message}}}' - pipeline: + tag: pipeline_b038445a name: '{{ IngestPipeline "pipeline_process_object" }}' on_failure: - set: @@ -28,3 +33,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_tasknameforpid.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_tasknameforpid.yml index 8851d5a50af..4018cb45f2f 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_tasknameforpid.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_tasknameforpid.yml @@ -2,27 +2,32 @@ description: Pipeline for Jamf Compliance Reporter aue_tasknameforpid audit logs. processors: - convert: + tag: convert_json_arguments_process_to_jamf_compliance_reporter_log_arguments_process_ee3dbde9 field: json.arguments.process target_field: jamf_compliance_reporter.log.arguments.process type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_arguments_target_port_to_jamf_compliance_reporter_log_arguments_target_port_cc0dcd91 field: json.arguments.target_port target_field: jamf_compliance_reporter.log.arguments.target.port type: long ignore_missing: true on_failure: - append: + tag: append_error_message_64d1e576 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_arguments_task_port_to_jamf_compliance_reporter_log_arguments_task_port_2fe271c9 field: json.arguments.task_port target_field: jamf_compliance_reporter.log.arguments.task.port type: long ignore_missing: true on_failure: - append: + tag: append_error_message_ddba9a56 field: error.message value: '{{{_ingest.on_failure_message}}}' on_failure: @@ -32,3 +37,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_unmount.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_unmount.yml index 4d55ece2e1d..c82e395c4d6 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_unmount.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_unmount.yml @@ -2,69 +2,83 @@ description: Pipeline for Jamf Compliance Reporter aue_unmount audit logs. processors: - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' - pipeline: + tag: pipeline_75de012d name: '{{ IngestPipeline "pipeline_exec_chain_child_object" }}' - rename: + tag: rename_json_path_to_jamf_compliance_reporter_log_path_dcaf10e2 field: json.path target_field: jamf_compliance_reporter.log.path ignore_missing: true - convert: + tag: convert_json_attributes_device_to_jamf_compliance_reporter_log_attributes_device_d89a67d9 field: json.attributes.device target_field: jamf_compliance_reporter.log.attributes.device type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_attributes_file_access_mode_to_json_file_access_mode_58473c4f field: json.attributes.file_access_mode target_field: json.file_access_mode ignore_missing: true - convert: + tag: convert_json_attributes_file_system_id_to_jamf_compliance_reporter_log_attributes_file_system_id_cc879847 field: json.attributes.file_system_id target_field: jamf_compliance_reporter.log.attributes.file.system.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_attributes_node_id_to_jamf_compliance_reporter_log_attributes_node_id_38b0771e field: json.attributes.node_id target_field: jamf_compliance_reporter.log.attributes.node.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_attributes_owner_group_id_to_jamf_compliance_reporter_log_attributes_owner_group_id_0c9a22e5 field: json.attributes.owner_group_id target_field: jamf_compliance_reporter.log.attributes.owner.group.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_attributes_owner_group_name_to_jamf_compliance_reporter_log_attributes_owner_group_name_580265ea field: json.attributes.owner_group_name target_field: jamf_compliance_reporter.log.attributes.owner.group.name ignore_missing: true - convert: + tag: convert_json_attributes_owner_user_id_58bbc519 field: json.attributes.owner_user_id type: string ignore_missing: true ignore_failure: true - append: + tag: append_user_id_f10ea64d field: user.id value: '{{{json.attributes.owner_user_id}}}' if: ctx.json?.attributes?.owner_user_id != null allow_duplicates: false ignore_failure: true - append: + tag: append_user_name_2ba08467 field: user.name value: '{{{json.attributes.owner_user_name}}}' if: ctx.json?.attributes?.owner_user_name != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_0321d5d7 field: related.user value: '{{{json.attributes.owner_user_name}}}' if: ctx.json?.attributes?.owner_user_name != null allow_duplicates: false ignore_failure: true - script: + tag: script_fbc9f1e5 description: Convert Decimal into Octal. lang: painless if: ctx.json?.file_access_mode != null @@ -78,3 +92,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_wait4.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_wait4.yml index 3b563287487..cec75cb0452 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_wait4.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_aue_wait4.yml @@ -2,15 +2,18 @@ description: Pipeline for Jamf Compliance Reporter aue_wait4 audit logs. processors: - convert: + tag: convert_json_arguments_pid_to_process_pid_bd6d088b field: json.arguments.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_5c0cdd9e field: error.message value: '{{{_ingest.on_failure_message}}}' - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' on_failure: - set: @@ -19,3 +22,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_compliance_reporter_tamper_event_and_file_event_info.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_compliance_reporter_tamper_event_and_file_event_info.yml index d5bdcaa81ac..f68ad164fa2 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_compliance_reporter_tamper_event_and_file_event_info.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_compliance_reporter_tamper_event_and_file_event_info.yml @@ -2,233 +2,284 @@ description: Pipeline for Jamf Compliance Reporter temper event and file event info event logs. processors: - convert: + tag: convert_json_file_event_info_eventid_wrapped_to_jamf_compliance_reporter_log_file_event_info_eventid_wrapped_162db86a field: json.file_event_info.eventid_wrapped target_field: jamf_compliance_reporter.log.file_event_info.eventid_wrapped type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_1bd52cbf field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_file_event_info_hash_to_file_hash_sha1_8e602c8f field: json.file_event_info.hash target_field: file.hash.sha1 ignore_missing: true - append: + tag: append_related_hash_b7dc7c76 field: related.hash value: '{{{file.hash.sha1}}}' if: ctx.file?.hash?.sha1 != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_file_event_info_history_done_to_jamf_compliance_reporter_log_file_event_info_history_done_eaaa513c field: json.file_event_info.history_done target_field: jamf_compliance_reporter.log.file_event_info.history_done type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_0d222565 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_item_change_owner_to_jamf_compliance_reporter_log_file_event_info_item_change_owner_817e23fb field: json.file_event_info.item_change_owner target_field: jamf_compliance_reporter.log.file_event_info.item.change_owner type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_77b32926 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_item_cloned_to_jamf_compliance_reporter_log_file_event_info_item_cloned_71c028d1 field: json.file_event_info.item_cloned target_field: jamf_compliance_reporter.log.file_event_info.item.cloned type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_997f36cc field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_item_created_to_jamf_compliance_reporter_log_file_event_info_item_created_e5634d61 field: json.file_event_info.item_created target_field: jamf_compliance_reporter.log.file_event_info.item.created type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_b02e1a80 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_item_extended_attribute_modified_to_jamf_compliance_reporter_log_file_event_info_item_extended_attribute_modified_96b50975 field: json.file_event_info.item_extended_attribute_modified target_field: jamf_compliance_reporter.log.file_event_info.item.extended_attribute_modified type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_7193db24 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_item_finder_info_modified_to_jamf_compliance_reporter_log_file_event_info_item_finder_info_modified_a76238a1 field: json.file_event_info.item_finder_info_modified target_field: jamf_compliance_reporter.log.file_event_info.item.finder_info_modified type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_9815cbb0 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_item_inode_metadata_modified_to_jamf_compliance_reporter_log_file_event_info_item_inode_metadata_modified_5308b7f5 field: json.file_event_info.item_inode_metadata_modified target_field: jamf_compliance_reporter.log.file_event_info.item.inode_metadata_modified type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_5fb63374 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_item_is_directory_to_jamf_compliance_reporter_log_file_event_info_item_is_directory_4988ed2b field: json.file_event_info.item_is_directory target_field: jamf_compliance_reporter.log.file_event_info.item.is_directory type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_e9f16296 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_item_is_file_to_jamf_compliance_reporter_log_file_event_info_item_is_file_76558e61 field: json.file_event_info.item_is_file target_field: jamf_compliance_reporter.log.file_event_info.item.is_file type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_d04b1fa0 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_item_is_hard_link_to_jamf_compliance_reporter_log_file_event_info_item_is_hard_link_48268001 field: json.file_event_info.item_is_hard_link target_field: jamf_compliance_reporter.log.file_event_info.item.is_hard_link type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_b24a75d0 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_item_is_last_hard_link_to_jamf_compliance_reporter_log_file_event_info_item_is_last_hard_link_a944ac45 field: json.file_event_info.item_is_last_hard_link target_field: jamf_compliance_reporter.log.file_event_info.item.is_last_hard_link type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_6b8d70c4 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_item_is_sym_link_to_jamf_compliance_reporter_log_file_event_info_item_is_sym_link_5df927bd field: json.file_event_info.item_is_sym_link target_field: jamf_compliance_reporter.log.file_event_info.item.is_sym_link type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_e8db4354 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_item_removed_to_jamf_compliance_reporter_log_file_event_info_item_removed_8911de3d field: json.file_event_info.item_removed target_field: jamf_compliance_reporter.log.file_event_info.item.removed type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_5258b5b4 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_item_renamed_to_jamf_compliance_reporter_log_file_event_info_item_renamed_3922d02d field: json.file_event_info.item_renamed target_field: jamf_compliance_reporter.log.file_event_info.item.renamed type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_7e108f1c field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_item_updated_to_jamf_compliance_reporter_log_file_event_info_item_updated_16bb2b39 field: json.file_event_info.item_updated target_field: jamf_compliance_reporter.log.file_event_info.item.updated type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_d497e650 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_kernel_dropped_to_jamf_compliance_reporter_log_file_event_info_kernel_dropped_24512574 field: json.file_event_info.kernel_dropped target_field: jamf_compliance_reporter.log.file_event_info.kernel_dropped type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_0e1a42cd field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_mount_to_jamf_compliance_reporter_log_file_event_info_mount_64e9d266 field: json.file_event_info.mount target_field: jamf_compliance_reporter.log.file_event_info.mount type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_1daba09f field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_must_scan_sub_dir_to_jamf_compliance_reporter_log_file_event_info_must_scan_sub_dir_fd61a1cc field: json.file_event_info.must_scan_sub_dir target_field: jamf_compliance_reporter.log.file_event_info.must_scan_sub_dir type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_355f5e21 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_none_to_jamf_compliance_reporter_log_file_event_info_none_570c5dfc field: json.file_event_info.none target_field: jamf_compliance_reporter.log.file_event_info.none type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_9520aed5 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_own_event_to_jamf_compliance_reporter_log_file_event_info_own_event_ddc1ce9e field: json.file_event_info.own_event target_field: jamf_compliance_reporter.log.file_event_info.own_event type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_b9cfe3cf field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_file_event_info_path_to_file_path_e7dea668 field: json.file_event_info.path target_field: file.path ignore_missing: true - convert: + tag: convert_json_file_event_info_root_changed_to_jamf_compliance_reporter_log_file_event_info_root_changed_052e638c field: json.file_event_info.root_changed target_field: jamf_compliance_reporter.log.file_event_info.root_changed type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_1b3fc4f5 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_unmount_to_jamf_compliance_reporter_log_file_event_info_unmount_4ac804f4 field: json.file_event_info.unmount target_field: jamf_compliance_reporter.log.file_event_info.unmount type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_97aeaa95 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_file_event_info_user_dropped_to_jamf_compliance_reporter_log_file_event_info_user_dropped_1409cc3c field: json.file_event_info.user_dropped target_field: jamf_compliance_reporter.log.file_event_info.user_dropped type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_b558c0d5 field: error.message value: '{{{_ingest.on_failure_message}}}' on_failure: @@ -238,3 +289,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_event.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_event.yml index 4b67f3ca4f6..fd64e905158 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_event.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_event.yml @@ -2,126 +2,160 @@ description: Pipeline for Jamf Compliance Reporter event logs. processors: - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_jamf_compliance_reporter_log_dataset_cfa1b036 field: jamf_compliance_reporter.log.dataset value: event - append: + tag: append_event_category_8a525c81 field: event.category value: process - set: + tag: set_host_os_type_074dd92b field: host.os.type value: macos - append: + tag: append_event_type_1057d615 field: event.type value: info if: '!["UNIFIED_LOG_EVENT","XPROTECT_EVENT_LOG"].contains(ctx.json?.header?.event_name)' - convert: + tag: convert_json__event_score_to_jamf_compliance_reporter_log_event_score_3cb39b47 field: json._event_score target_field: jamf_compliance_reporter.log.event_score type: long ignore_missing: true on_failure: - append: + tag: append_error_message_58844c12 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_header_event_name_to_event_action_99289c09 field: json.header.event_name target_field: event.action ignore_missing: true - lowercase: + tag: lowercase_event_action_9334b869 field: event.action ignore_missing: true - date: + tag: date_json_header_time_seconds_epoch_726cfcbc field: json.header.time_seconds_epoch if: ctx.json?.header?.time_seconds_epoch != 0 formats: - UNIX on_failure: - append: + tag: append_error_message_3be7c1d1 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_host_info_host_name_to_host_hostname_b155f2aa field: json.host_info.host_name target_field: host.hostname ignore_missing: true - append: + tag: append_related_hosts_e6462550 field: related.hosts value: '{{{host.hostname}}}' if: ctx.host?.hostname != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_host_info_host_uuid_to_jamf_compliance_reporter_log_host_info_host_uuid_73fc6247 field: json.host_info.host_uuid target_field: jamf_compliance_reporter.log.host_info.host.uuid ignore_missing: true - rename: + tag: rename_json_host_info_osversion_to_host_os_version_d3d10e97 field: json.host_info.osversion target_field: host.os.version ignore_missing: true - append: + tag: append_host_mac_ddef4d66 field: host.mac value: '{{{json.host_info.primary_mac_address}}}' if: ctx.json?.host_info?.primary_mac_address != null allow_duplicates: false ignore_failure: true - gsub: + tag: gsub_host_mac_d1faae6a field: host.mac pattern: '[-:.]' replacement: '-' ignore_missing: true - uppercase: + tag: uppercase_host_mac_73cd4413 field: host.mac ignore_missing: true - rename: + tag: rename_json_host_info_serial_number_to_host_id_02030592 field: json.host_info.serial_number target_field: host.id ignore_missing: true # NOTE: ctx.event is initialized by an earlier processor - pipeline: + tag: pipeline_4e8e6521 name: '{{ IngestPipeline "pipeline_audio_video_device_event" }}' if: ctx.event.action == 'audio_video_device_event' - pipeline: + tag: pipeline_25e1db21 name: '{{ IngestPipeline "pipeline_audit_class_verification_event" }}' if: ctx.event.action == 'audit_class_verification_event' - pipeline: + tag: pipeline_d051078b name: '{{ IngestPipeline "pipeline_compliance_reporter_tamper_event_and_file_event_info" }}' if: '["compliance_reporter_tamper_event", "file_event"].contains(ctx.event.action)' - pipeline: + tag: pipeline_4f37cfd5 name: '{{ IngestPipeline "pipeline_gatekeeper_info_event" }}' if: ctx.event.action == 'gatekeeper_info_event' - pipeline: + tag: pipeline_a21c17ab name: '{{ IngestPipeline "pipeline_gatekeeper_manual_overrides" }}' if: ctx.event.action == 'gatekeeper_manual_overrides' - pipeline: + tag: pipeline_7deb2ac5 name: '{{ IngestPipeline "pipeline_gatekeeper_quarantine_log" }}' if: ctx.event.action == 'gatekeeper_quarantine_log' - pipeline: + tag: pipeline_a993ada5 name: '{{ IngestPipeline "pipeline_hardware_event" }}' if: ctx.event.action == 'hardware_event' - pipeline: + tag: pipeline_4892e065 name: '{{ IngestPipeline "pipeline_license_info_event" }}' if: ctx.event.action == 'license_info_event' - pipeline: + tag: pipeline_29a77bf5 name: '{{ IngestPipeline "pipeline_preference_list_event" }}' if: ctx.event.action == 'preference_list_event' - pipeline: + tag: pipeline_d9d5eaf5 name: '{{ IngestPipeline "pipeline_print_event_information" }}' if: ctx.event.action == 'print_event_information' - pipeline: + tag: pipeline_8898d77d name: '{{ IngestPipeline "pipeline_prohibited_app_blocked" }}' if: ctx.event.action == 'prohibited_app_blocked' - pipeline: + tag: pipeline_7152110d name: '{{ IngestPipeline "pipeline_signal_event" }}' if: ctx.event.action == 'signal_event' - pipeline: + tag: pipeline_72f5a3d3 name: '{{ IngestPipeline "pipeline_unified_log_event" }}' if: ctx.event.action == 'unified_log_event' - pipeline: + tag: pipeline_6f8ee6e3 name: '{{ IngestPipeline "pipeline_xprotect_definitions_version_info" }}' if: ctx.event.action == 'xprotect_definitions_version_info' - pipeline: + tag: pipeline_eb72550d name: '{{ IngestPipeline "pipeline_xprotect_event_log" }}' if: ctx.event.action == 'xprotect_event_log' on_failure: @@ -131,3 +165,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_exec_chain_child_object.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_exec_chain_child_object.yml index a4b385c83a0..0bec10ff536 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_exec_chain_child_object.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_exec_chain_child_object.yml @@ -2,19 +2,23 @@ description: Pipeline for Jamf Compliance Reporter exec_chain_child_object audit logs. processors: - rename: + tag: rename_json_exec_chain_child_parent_path_to_jamf_compliance_reporter_log_exec_chain_child_parent_path_4347ae49 field: json.exec_chain_child.parent_path target_field: jamf_compliance_reporter.log.exec_chain_child.parent.path ignore_missing: true - convert: + tag: convert_json_exec_chain_child_parent_pid_to_process_parent_pid_409cd750 field: json.exec_chain_child.parent_pid target_field: process.parent.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_ea707b8b field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_exec_chain_child_parent_uuid_to_jamf_compliance_reporter_log_exec_chain_child_parent_uuid_b1d29201 field: json.exec_chain_child.parent_uuid target_field: jamf_compliance_reporter.log.exec_chain_child.parent.uuid ignore_missing: true @@ -25,3 +29,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_gatekeeper_info_event.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_gatekeeper_info_event.yml index 2e54976687e..45cadffae57 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_gatekeeper_info_event.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_gatekeeper_info_event.yml @@ -2,28 +2,34 @@ description: Pipeline for Jamf Compliance Reporter gatekeeper info event logs. processors: - convert: + tag: convert_json_event_attributes_assessments_enabled_to_jamf_compliance_reporter_log_event_attributes_assessments_enabled_359ee9f2 field: json.event_attributes.assessments_enabled target_field: jamf_compliance_reporter.log.event_attributes.assessments_enabled type: long ignore_missing: true on_failure: - append: + tag: append_error_message_99ed890d field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_event_attributes_dev_id_enabled_to_jamf_compliance_reporter_log_event_attributes_dev_id_enabled_e7447490 field: json.event_attributes.dev_id_enabled target_field: jamf_compliance_reporter.log.event_attributes.dev_id_enabled type: long ignore_missing: true on_failure: - append: + tag: append_error_message_aa03b2df field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_event_attributes_opaque_version_to_jamf_compliance_reporter_log_event_attributes_opaque_version_0e7ca4e6 field: json.event_attributes.opaque_version target_field: jamf_compliance_reporter.log.event_attributes.opaque_version ignore_missing: true - rename: + tag: rename_json_event_attributes_version_to_jamf_compliance_reporter_log_event_attributes_version_4c020ad0 field: json.event_attributes.version target_field: jamf_compliance_reporter.log.event_attributes.version ignore_missing: true @@ -34,3 +40,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_gatekeeper_manual_overrides.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_gatekeeper_manual_overrides.yml index bcfefa75862..1c931375348 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_gatekeeper_manual_overrides.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_gatekeeper_manual_overrides.yml @@ -2,6 +2,7 @@ description: Pipeline for Jamf Compliance Reporter gatekeeper manual overrides event logs. processors: - foreach: + tag: foreach_json_event_attributes_attributes_f28d3ed4 field: json.event_attributes.attributes if: ctx.json?.event_attributes?.attributes instanceof List processor: @@ -18,6 +19,7 @@ processors: field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_event_attributes_attributes_6c662502 field: json.event_attributes.attributes if: ctx.json?.event_attributes?.attributes instanceof List processor: @@ -34,10 +36,12 @@ processors: field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_event_attributes_attributes_to_jamf_compliance_reporter_log_event_attributes_attributes_12fda942 field: json.event_attributes.attributes target_field: jamf_compliance_reporter.log.event_attributes.attributes ignore_missing: true - rename: + tag: rename_json_event_attributes_path_to_jamf_compliance_reporter_log_event_attributes_path_f06dbbf2 field: json.event_attributes.path target_field: jamf_compliance_reporter.log.event_attributes.path ignore_missing: true @@ -48,3 +52,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_gatekeeper_quarantine_log.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_gatekeeper_quarantine_log.yml index f0de852db5d..de26e848ad3 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_gatekeeper_quarantine_log.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_gatekeeper_quarantine_log.yml @@ -2,6 +2,7 @@ description: Pipeline for Jamf Compliance Reporter gatekeeper quarantine event logs. processors: - foreach: + tag: foreach_json_event_attributes_attributes_4045f89c field: json.event_attributes.attributes if: ctx.json?.event_attributes?.attributes instanceof List processor: @@ -10,6 +11,7 @@ processors: target_field: _ingest._value.quarantine.agent_bundle_identifier ignore_failure: true - foreach: + tag: foreach_json_event_attributes_attributes_eb2970c3 field: json.event_attributes.attributes if: ctx.json?.event_attributes?.attributes instanceof List processor: @@ -18,6 +20,7 @@ processors: target_field: _ingest._value.quarantine.agent_name ignore_failure: true - foreach: + tag: foreach_json_event_attributes_attributes_328ffe70 field: json.event_attributes.attributes if: ctx.json?.event_attributes?.attributes instanceof List processor: @@ -26,6 +29,7 @@ processors: target_field: _ingest._value.quarantine.data_url_string ignore_failure: true - foreach: + tag: foreach_json_event_attributes_attributes_1b46db15 field: json.event_attributes.attributes if: ctx.json?.event_attributes?.attributes instanceof List processor: @@ -34,6 +38,7 @@ processors: target_field: _ingest._value.quarantine.event_identifier ignore_failure: true - foreach: + tag: foreach_json_event_attributes_attributes_16239074 field: json.event_attributes.attributes if: ctx.json?.event_attributes?.attributes instanceof List processor: @@ -42,6 +47,7 @@ processors: target_field: _ingest._value.quarantine.origin_url_string ignore_failure: true - foreach: + tag: foreach_json_event_attributes_attributes_177014c6 field: json.event_attributes.attributes if: ctx.json?.event_attributes?.attributes instanceof List processor: @@ -58,6 +64,7 @@ processors: field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_event_attributes_attributes_affd4261 field: json.event_attributes.attributes if: ctx.json?.event_attributes?.attributes instanceof List processor: @@ -65,10 +72,12 @@ processors: field: _ingest._value.QuarantineTimeStamp ignore_failure: true - rename: + tag: rename_json_event_attributes_attributes_to_jamf_compliance_reporter_log_event_attributes_attributes_12fda942 field: json.event_attributes.attributes target_field: jamf_compliance_reporter.log.event_attributes.attributes ignore_missing: true - rename: + tag: rename_json_event_attributes_path_to_jamf_compliance_reporter_log_event_attributes_path_f06dbbf2 field: json.event_attributes.path target_field: jamf_compliance_reporter.log.event_attributes.path ignore_missing: true @@ -79,3 +88,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_hardware_event.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_hardware_event.yml index c03ab262112..e997776b93c 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_hardware_event.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_hardware_event.yml @@ -2,85 +2,104 @@ description: Pipeline for Jamf Compliance Reporter hardware event logs. processors: - rename: + tag: rename_json_hardware_event_info_device_attributes_IOCFPlugInTypes_to_jamf_compliance_reporter_log_hardware_event_info_device_attributes_io_cf_plugin_types_03f76ec6 field: json.hardware_event_info.device_attributes.IOCFPlugInTypes target_field: jamf_compliance_reporter.log.hardware_event_info.device_attributes.io.cf_plugin_types ignore_missing: true - rename: + tag: rename_json_hardware_event_info_device_attributes_IOClassNameOverride_to_jamf_compliance_reporter_log_hardware_event_info_device_attributes_io_class_name_override_47e6b44e field: json.hardware_event_info.device_attributes.IOClassNameOverride target_field: jamf_compliance_reporter.log.hardware_event_info.device_attributes.io.class_name_override ignore_missing: true - convert: + tag: convert_json_hardware_event_info_device_attributes_IOPowerManagement_CapabilityFlags_to_jamf_compliance_reporter_log_hardware_event_info_device_attributes_io_power_management_capability_flags_ae811b3b field: json.hardware_event_info.device_attributes.IOPowerManagement.CapabilityFlags target_field: jamf_compliance_reporter.log.hardware_event_info.device_attributes.io.power_management.capability_flags type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_hardware_event_info_device_attributes_IOPowerManagement_CurrentPowerState_to_jamf_compliance_reporter_log_hardware_event_info_device_attributes_io_power_management_current_power_state_bf762819 field: json.hardware_event_info.device_attributes.IOPowerManagement.CurrentPowerState target_field: jamf_compliance_reporter.log.hardware_event_info.device_attributes.io.power_management.current_power_state type: long ignore_missing: true on_failure: - append: + tag: append_error_message_0da3f8e6 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_hardware_event_info_device_attributes_IOPowerManagement_DevicePowerState_to_jamf_compliance_reporter_log_hardware_event_info_device_attributes_io_power_management_device_power_state_a1552d65 field: json.hardware_event_info.device_attributes.IOPowerManagement.DevicePowerState target_field: jamf_compliance_reporter.log.hardware_event_info.device_attributes.io.power_management.device_power_state type: long ignore_missing: true on_failure: - append: + tag: append_error_message_c51c7b3a field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_hardware_event_info_device_attributes_IOPowerManagement_DriverPowerState_to_jamf_compliance_reporter_log_hardware_event_info_device_attributes_io_power_management_driver_power_state_8412dfd9 field: json.hardware_event_info.device_attributes.IOPowerManagement.DriverPowerState target_field: jamf_compliance_reporter.log.hardware_event_info.device_attributes.io.power_management.driver_power_state type: long ignore_missing: true on_failure: - append: + tag: append_error_message_3888827e field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_hardware_event_info_device_attributes_IOPowerManagement_MaxPowerState_to_jamf_compliance_reporter_log_hardware_event_info_device_attributes_io_power_management_max_power_state_c56a2223 field: json.hardware_event_info.device_attributes.IOPowerManagement.MaxPowerState target_field: jamf_compliance_reporter.log.hardware_event_info.device_attributes.io.power_management.max_power_state type: long ignore_missing: true on_failure: - append: + tag: append_error_message_cad763f4 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_hardware_event_info_device_attributes_Removable_to_jamf_compliance_reporter_log_hardware_event_info_device_attributes_removable_e4afd4d2 field: json.hardware_event_info.device_attributes.Removable target_field: jamf_compliance_reporter.log.hardware_event_info.device_attributes.removable ignore_missing: true - rename: + tag: rename_json_hardware_event_info_device_attributes_USB_Product_Name_to_jamf_compliance_reporter_log_hardware_event_info_device_attributes_usb_product_name_deb969b5 field: json.hardware_event_info.device_attributes.USB Product Name target_field: jamf_compliance_reporter.log.hardware_event_info.device_attributes.usb.product_name ignore_missing: true - rename: + tag: rename_json_hardware_event_info_device_attributes_USB_Vendor_Name_to_jamf_compliance_reporter_log_hardware_event_info_device_attributes_usb_vendor_name_f4008447 field: json.hardware_event_info.device_attributes.USB Vendor Name target_field: jamf_compliance_reporter.log.hardware_event_info.device_attributes.usb.vendor_name ignore_missing: true - convert: + tag: convert_json_hardware_event_info_device_attributes_iSerialNumber_to_jamf_compliance_reporter_log_hardware_event_info_device_attributes_iserial_number_3e67899b field: json.hardware_event_info.device_attributes.iSerialNumber target_field: jamf_compliance_reporter.log.hardware_event_info.device_attributes.iserial_number type: long ignore_missing: true on_failure: - append: + tag: append_error_message_7ef7f4da field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_hardware_event_info_device_class_to_jamf_compliance_reporter_log_hardware_event_info_device_class_550796f1 field: json.hardware_event_info.device_class target_field: jamf_compliance_reporter.log.hardware_event_info.device.class ignore_missing: true - rename: + tag: rename_json_hardware_event_info_device_name_to_jamf_compliance_reporter_log_hardware_event_info_device_name_932bb5bb field: json.hardware_event_info.device_name target_field: jamf_compliance_reporter.log.hardware_event_info.device.name ignore_missing: true - rename: + tag: rename_json_hardware_event_info_device_status_to_jamf_compliance_reporter_log_hardware_event_info_device_status_669027c3 field: json.hardware_event_info.device_status target_field: jamf_compliance_reporter.log.hardware_event_info.device.status ignore_missing: true @@ -91,3 +110,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_identity_object.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_identity_object.yml index 0ce585d32d3..59d31d93c91 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_identity_object.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_identity_object.yml @@ -2,36 +2,43 @@ description: Pipeline for Jamf Compliance Reporter identity_object audit logs. processors: - rename: + tag: rename_json_identity_cd_hash_to_jamf_compliance_reporter_log_identity_cd_hash_a91be7b4 field: json.identity.cd_hash target_field: jamf_compliance_reporter.log.identity.cd_hash ignore_missing: true - append: + tag: append_related_hash_04fe9d31 field: related.hash value: '{{{jamf_compliance_reporter.log.identity.cd_hash}}}' if: ctx.jamf_compliance_reporter?.log?.identity?.cd_hash != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_identity_signer_id_to_jamf_compliance_reporter_log_identity_signer_id_8ee25a7d field: json.identity.signer_id target_field: jamf_compliance_reporter.log.identity.signer.id ignore_missing: true - convert: + tag: convert_json_identity_signer_id_truncated_to_jamf_compliance_reporter_log_identity_signer_id_truncated_7d4200ce field: json.identity.signer_id_truncated target_field: jamf_compliance_reporter.log.identity.signer.id_truncated type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_identity_signer_type_to_jamf_compliance_reporter_log_identity_signer_type_779a9f36 field: json.identity.signer_type target_field: jamf_compliance_reporter.log.identity.signer.type type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_identity_team_id_to_jamf_compliance_reporter_log_identity_team_id_cc9f74b3 field: json.identity.team_id target_field: jamf_compliance_reporter.log.identity.team.id ignore_missing: true - convert: + tag: convert_json_identity_team_id_truncated_to_jamf_compliance_reporter_log_identity_team_id_truncated_20bad62c field: json.identity.team_id_truncated target_field: jamf_compliance_reporter.log.identity.team.id_truncated type: string @@ -44,3 +51,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_license_info_event.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_license_info_event.yml index cfca45d4f75..38f8ebb1f5c 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_license_info_event.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_license_info_event.yml @@ -2,16 +2,19 @@ description: Pipeline for Jamf Compliance Reporter license info event logs. processors: - rename: + tag: rename_json_ComplianceReporter_license_info_email_to_user_email_f040c868 field: json.ComplianceReporter_license_info.email target_field: user.email ignore_missing: true - append: + tag: append_related_user_36d4b55a field: related.user value: '{{{user.email}}}' if: ctx.user?.email != null allow_duplicates: false ignore_failure: true - date: + tag: date_json_ComplianceReporter_license_info_expiration_date_to_jamf_compliance_reporter_log_compliancereporter_license_info_expiration_date_920484de field: json.ComplianceReporter_license_info.expiration_date target_field: jamf_compliance_reporter.log.compliancereporter_license_info.expiration_date if: ctx.json?.ComplianceReporter_license_info?.expiration_date != null && ctx.json.ComplianceReporter_license_info.expiration_date != 0 @@ -19,13 +22,16 @@ processors: - dd/MM/yyyy on_failure: - append: + tag: append_error_message_77b9a54b field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_ComplianceReporter_license_info_status_to_jamf_compliance_reporter_log_compliancereporter_license_info_status_3d065462 field: json.ComplianceReporter_license_info.status target_field: jamf_compliance_reporter.log.compliancereporter_license_info.status ignore_missing: true - date: + tag: date_json_ComplianceReporter_license_info_time_seconds_epoch_to_jamf_compliance_reporter_log_compliancereporter_license_info_time_89cd2f48 field: json.ComplianceReporter_license_info.time_seconds_epoch target_field: jamf_compliance_reporter.log.compliancereporter_license_info.time if: ctx.json?.ComplianceReporter_license_info?.time_seconds_epoch != null && ctx.json.ComplianceReporter_license_info.time_seconds_epoch != '0' @@ -33,13 +39,16 @@ processors: - UNIX on_failure: - append: + tag: append_error_message_99611759 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_ComplianceReporter_license_info_type_to_jamf_compliance_reporter_log_compliancereporter_license_info_type_759c18be field: json.ComplianceReporter_license_info.type target_field: jamf_compliance_reporter.log.compliancereporter_license_info.type ignore_missing: true - rename: + tag: rename_json_ComplianceReporter_license_info_version_to_jamf_compliance_reporter_log_compliancereporter_license_info_version_34b4ddce field: json.ComplianceReporter_license_info.version target_field: jamf_compliance_reporter.log.compliancereporter_license_info.version ignore_missing: true @@ -50,3 +59,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_preference_list_event.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_preference_list_event.yml index 178b0e36030..bb057ad4370 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_preference_list_event.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_preference_list_event.yml @@ -2,56 +2,68 @@ description: Pipeline for Jamf Compliance Reporter preference list event logs. processors: - rename: + tag: rename_json_event_attributes_AuditEventExcludedProcesses_to_jamf_compliance_reporter_log_event_attributes_audit_event_excluded_processes_5db654ce field: json.event_attributes.AuditEventExcludedProcesses target_field: jamf_compliance_reporter.log.event_attributes.audit_event.excluded_processes ignore_missing: true - rename: + tag: rename_json_event_attributes_AuditEventExcludedUsers_to_jamf_compliance_reporter_log_event_attributes_audit_event_excluded_users_7f00e1f6 field: json.event_attributes.AuditEventExcludedUsers target_field: jamf_compliance_reporter.log.event_attributes.audit_event.excluded_users ignore_missing: true - convert: + tag: convert_json_event_attributes_AuditEventLogVerboseMessages_to_jamf_compliance_reporter_log_event_attributes_audit_event_log_verbose_messages_80be1edd field: json.event_attributes.AuditEventLogVerboseMessages target_field: jamf_compliance_reporter.log.event_attributes.audit_event_log_verbose_messages type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_event_attributes_AuditLevel_to_jamf_compliance_reporter_log_event_attributes_audit_level_d73b32a5 field: json.event_attributes.AuditLevel target_field: jamf_compliance_reporter.log.event_attributes.audit_level type: long ignore_missing: true on_failure: - append: + tag: append_error_message_6dece8a8 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_event_attributes_FileEventExclusionPaths_to_jamf_compliance_reporter_log_event_attributes_file_event_exclusion_paths_91a1d1e2 field: json.event_attributes.FileEventExclusionPaths target_field: jamf_compliance_reporter.log.event_attributes.file_event.exclusion_paths ignore_missing: true - rename: + tag: rename_json_event_attributes_FileEventInclusionPaths_to_jamf_compliance_reporter_log_event_attributes_file_event_inclusion_paths_28bcb772 field: json.event_attributes.FileEventInclusionPaths target_field: jamf_compliance_reporter.log.event_attributes.file_event.inclusion_paths ignore_missing: true - convert: + tag: convert_json_event_attributes_FileEventUseFuzzyMatch_to_jamf_compliance_reporter_log_event_attributes_file_event_use_fuzzy_match_313a8a91 field: json.event_attributes.FileEventUseFuzzyMatch target_field: jamf_compliance_reporter.log.event_attributes.file_event.use_fuzzy_match type: long ignore_missing: true on_failure: - append: + tag: append_error_message_aa345a1e field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_event_attributes_FileLicenseInfo_LicenseEmail_to_user_email_cecfad41 field: json.event_attributes.FileLicenseInfo.LicenseEmail target_field: user.email ignore_missing: true - append: + tag: append_related_user_36d4b55a field: related.user value: '{{{user.email}}}' if: ctx.user?.email != null allow_duplicates: false ignore_failure: true - date: + tag: date_json_event_attributes_FileLicenseInfo_LicenseExpirationDate_to_jamf_compliance_reporter_log_event_attributes_file_license_info_license_expiration_date_c1e20f36 field: json.event_attributes.FileLicenseInfo.LicenseExpirationDate target_field: jamf_compliance_reporter.log.event_attributes.file_license_info.license_expiration_date if: ctx.json?.event_attributes?.FileLicenseInfo?.LicenseExpirationDate != '0' @@ -59,83 +71,103 @@ processors: - dd/MM/yyyy on_failure: - append: + tag: append_error_message_d75047b7 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_event_attributes_FileLicenseInfo_LicenseKey_to_jamf_compliance_reporter_log_event_attributes_file_license_info_license_key_f6197f8b field: json.event_attributes.FileLicenseInfo.LicenseKey target_field: jamf_compliance_reporter.log.event_attributes.file_license_info.license_key ignore_missing: true - rename: + tag: rename_json_event_attributes_FileLicenseInfo_LicenseType_to_jamf_compliance_reporter_log_event_attributes_file_license_info_license_type_03c1afcd field: json.event_attributes.FileLicenseInfo.LicenseType target_field: jamf_compliance_reporter.log.event_attributes.file_license_info.license_type ignore_missing: true - rename: + tag: rename_json_event_attributes_FileLicenseInfo_LicenseVersion_to_jamf_compliance_reporter_log_event_attributes_file_license_info_license_version_d374c275 field: json.event_attributes.FileLicenseInfo.LicenseVersion target_field: jamf_compliance_reporter.log.event_attributes.file_license_info.license_version ignore_missing: true - rename: + tag: rename_json_event_attributes_LogFileLocation_to_jamf_compliance_reporter_log_event_attributes_log_file_location_fc5fcc6a field: json.event_attributes.LogFileLocation target_field: jamf_compliance_reporter.log.event_attributes.log.file.location ignore_missing: true - convert: + tag: convert_json_event_attributes_LogFileMaxNumberBackups_to_jamf_compliance_reporter_log_event_attributes_log_file_max_number_backups_1356e702 field: json.event_attributes.LogFileMaxNumberBackups target_field: jamf_compliance_reporter.log.event_attributes.log.file.max_number_backups type: long ignore_missing: true on_failure: - append: + tag: append_error_message_cd6394f5 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_event_attributes_LogFileMaxSizeMegaBytes_to_jamf_compliance_reporter_log_event_attributes_log_file_max_size_mega_bytes_ee65d645 field: json.event_attributes.LogFileMaxSizeMegaBytes target_field: jamf_compliance_reporter.log.event_attributes.log.file.max_size_mega_bytes type: long ignore_missing: true on_failure: - append: + tag: append_error_message_a97b47b8 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_event_attributes_LogFileOwnership_to_jamf_compliance_reporter_log_event_attributes_log_file_ownership_352e7e8a field: json.event_attributes.LogFileOwnership target_field: jamf_compliance_reporter.log.event_attributes.log.file.ownership ignore_missing: true - rename: + tag: rename_json_event_attributes_LogFilePermission_to_jamf_compliance_reporter_log_event_attributes_log_file_permission_e9387f6a field: json.event_attributes.LogFilePermission target_field: jamf_compliance_reporter.log.event_attributes.log.file.permission ignore_missing: true - rename: + tag: rename_json_event_attributes_LogRemoteEndpointEnabled_to_jamf_compliance_reporter_log_event_attributes_log_remote_endpoint_enabled_bdc31d6c field: json.event_attributes.LogRemoteEndpointEnabled target_field: jamf_compliance_reporter.log.event_attributes.log.remote_endpoint_enabled ignore_missing: true - rename: + tag: rename_json_event_attributes_LogRemoteEndpointType_to_jamf_compliance_reporter_log_event_attributes_log_remote_endpoint_type_ba7fb706 field: json.event_attributes.LogRemoteEndpointType target_field: jamf_compliance_reporter.log.event_attributes.log.remote_endpoint_type ignore_missing: true - rename: + tag: rename_json_event_attributes_LogRemoteEndpointTypeAWSKinesis_AccessKeyId_to_jamf_compliance_reporter_log_event_attributes_log_remote_endpoint_type_awskinesis_access_key_id_dbd8cde9 field: json.event_attributes.LogRemoteEndpointTypeAWSKinesis.AccessKeyId target_field: jamf_compliance_reporter.log.event_attributes.log.remote_endpoint_type_awskinesis.access_key_id ignore_missing: true - rename: + tag: rename_json_event_attributes_LogRemoteEndpointTypeAWSKinesis_Region_to_jamf_compliance_reporter_log_event_attributes_log_remote_endpoint_type_awskinesis_region_abc8338f field: json.event_attributes.LogRemoteEndpointTypeAWSKinesis.Region target_field: jamf_compliance_reporter.log.event_attributes.log.remote_endpoint_type_awskinesis.region ignore_missing: true - rename: + tag: rename_json_event_attributes_LogRemoteEndpointTypeAWSKinesis_SecretKey_to_jamf_compliance_reporter_log_event_attributes_log_remote_endpoint_type_awskinesis_secret_key_b81f0970 field: json.event_attributes.LogRemoteEndpointTypeAWSKinesis.SecretKey target_field: jamf_compliance_reporter.log.event_attributes.log.remote_endpoint_type_awskinesis.secret_key ignore_missing: true - rename: + tag: rename_json_event_attributes_LogRemoteEndpointTypeAWSKinesis_StreamName_to_jamf_compliance_reporter_log_event_attributes_log_remote_endpoint_type_awskinesis_stream_name_8a4bd90c field: json.event_attributes.LogRemoteEndpointTypeAWSKinesis.StreamName target_field: jamf_compliance_reporter.log.event_attributes.log.remote_endpoint_type_awskinesis.stream_name ignore_missing: true - rename: + tag: rename_json_event_attributes_LogRemoteEndpointURL_to_jamf_compliance_reporter_log_event_attributes_log_remote_endpoint_url_fdc49950 field: json.event_attributes.LogRemoteEndpointURL target_field: jamf_compliance_reporter.log.event_attributes.log.remote_endpoint_url ignore_missing: true - rename: + tag: rename_json_event_attributes_UnifiedLogPredicates_to_jamf_compliance_reporter_log_event_attributes_unified_log_predicates_49395012 field: json.event_attributes.UnifiedLogPredicates target_field: jamf_compliance_reporter.log.event_attributes.unified_log_predicates ignore_missing: true - rename: + tag: rename_json_event_attributes_Version_to_jamf_compliance_reporter_log_event_attributes_version_383e1db0 field: json.event_attributes.Version target_field: jamf_compliance_reporter.log.event_attributes.version ignore_missing: true @@ -146,3 +178,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_print_event_information.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_print_event_information.yml index f61a2a417b5..995b00d20be 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_print_event_information.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_print_event_information.yml @@ -2,6 +2,7 @@ description: Pipeline for Jamf Compliance Reporter print information event logs. processors: - date: + tag: date_json_event_attributes_job_completed_time_to_jamf_compliance_reporter_log_event_attributes_job_completed_time_26abacc1 field: json.event_attributes.job_completed_time target_field: jamf_compliance_reporter.log.event_attributes.job.completed_time if: ctx.json?.event_attributes?.job_completed_time != null && ctx.json.event_attributes.job_completed_time != 0 @@ -9,9 +10,11 @@ processors: - UNIX on_failure: - append: + tag: append_error_message_3111d624 field: error.message value: '{{{_ingest.on_failure_message}}}' - date: + tag: date_json_event_attributes_job_creation_time_to_jamf_compliance_reporter_log_event_attributes_job_creation_time_5e043b2d field: json.event_attributes.job_creation_time target_field: jamf_compliance_reporter.log.event_attributes.job.creation_time if: ctx.json?.event_attributes?.job_creation_time != null && ctx.json.event_attributes.job_creation_time != 0 @@ -19,23 +22,28 @@ processors: - UNIX on_failure: - append: + tag: append_error_message_1a83db84 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_event_attributes_job_destination_to_jamf_compliance_reporter_log_event_attributes_job_destination_3029a517 field: json.event_attributes.job_destination target_field: jamf_compliance_reporter.log.event_attributes.job.destination ignore_missing: true - rename: + tag: rename_json_event_attributes_job_format_to_jamf_compliance_reporter_log_event_attributes_job_format_03a90183 field: json.event_attributes.job_format target_field: jamf_compliance_reporter.log.event_attributes.job.format ignore_missing: true - convert: + tag: convert_json_event_attributes_job_id_to_jamf_compliance_reporter_log_event_attributes_job_id_659f6918 field: json.event_attributes.job_id target_field: jamf_compliance_reporter.log.event_attributes.job.id type: string ignore_missing: true ignore_failure: true - date: + tag: date_json_event_attributes_job_processing_time_to_jamf_compliance_reporter_log_event_attributes_job_processing_time_2d03e949 field: json.event_attributes.job_processing_time target_field: jamf_compliance_reporter.log.event_attributes.job.processing_time if: ctx.json?.event_attributes?.job_processing_time != null && ctx.json.event_attributes.job_processing_time != 0 @@ -43,25 +51,31 @@ processors: - UNIX on_failure: - append: + tag: append_error_message_99f9646c field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_event_attributes_job_size_to_jamf_compliance_reporter_log_event_attributes_job_size_25fa88e7 field: json.event_attributes.job_size target_field: jamf_compliance_reporter.log.event_attributes.job.size ignore_missing: true - rename: + tag: rename_json_event_attributes_job_state_to_jamf_compliance_reporter_log_event_attributes_job_state_a6a6d749 field: json.event_attributes.job_state target_field: jamf_compliance_reporter.log.event_attributes.job.state ignore_missing: true - rename: + tag: rename_json_event_attributes_job_title_to_jamf_compliance_reporter_log_event_attributes_job_title_2141454b field: json.event_attributes.job_title target_field: jamf_compliance_reporter.log.event_attributes.job.title ignore_missing: true - rename: + tag: rename_json_event_attributes_job_user_to_jamf_compliance_reporter_log_event_attributes_job_user_057ba2cb field: json.event_attributes.job_user target_field: jamf_compliance_reporter.log.event_attributes.job.user ignore_missing: true - append: + tag: append_related_user_2c4b76af field: related.user value: '{{{jamf_compliance_reporter.log.event_attributes.job.user}}}' if: ctx.jamf_compliance_reporter?.log?.event_attributes?.job?.user != null @@ -74,3 +88,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_process_object.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_process_object.yml index a66ef2d8eeb..9c8a0528a66 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_process_object.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_process_object.yml @@ -2,143 +2,171 @@ description: Pipeline for Jamf process_object audit logs. processors: - convert: + tag: convert_json_process_audit_id_to_jamf_compliance_reporter_log_process_pid_22f4fd10 field: json.process.audit_id target_field: jamf_compliance_reporter.log.process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_f68be1a1 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_process_effective_group_id_to_jamf_compliance_reporter_log_process_effective_group_id_4b5a0d35 field: json.process.effective_group_id target_field: jamf_compliance_reporter.log.process.effective.group.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_process_effective_group_name_to_jamf_compliance_reporter_log_process_effective_group_name_7fc20b7c field: json.process.effective_group_name target_field: jamf_compliance_reporter.log.process.effective.group.name ignore_missing: true - convert: + tag: convert_json_process_effective_user_id_to_jamf_compliance_reporter_log_process_effective_user_id_923ff22b field: json.process.effective_user_id target_field: jamf_compliance_reporter.log.process.effective.user.id type: string ignore_missing: true ignore_failure: true - append: + tag: append_user_effective_id_79bcf892 field: user.effective.id value: '{{{json.process.effective_user_id}}}' if: ctx.json?.process?.effective_user_id != null allow_duplicates: false ignore_failure: true - append: + tag: append_user_effective_name_ad60ed3c field: user.effective.name value: '{{{json.process.effective_user_name}}}' if: ctx.json?.process?.effective_user_name != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_process_effective_user_name_to_jamf_compliance_reporter_log_process_effective_user_name_585005fc field: json.process.effective_user_name target_field: jamf_compliance_reporter.log.process.effective.user.name ignore_missing: true - append: + tag: append_related_user_d7dd8b1e field: related.user value: '{{{jamf_compliance_reporter.log.process.effective.user.name}}}' if: ctx.jamf_compliance_reporter?.log?.process?.effective?.user?.name != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_process_group_id_to_jamf_compliance_reporter_log_process_group_id_008b0044 field: json.process.group_id target_field: jamf_compliance_reporter.log.process.group.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_process_group_name_to_jamf_compliance_reporter_log_process_group_name_517acdf9 field: json.process.group_name target_field: jamf_compliance_reporter.log.process.group.name ignore_missing: true - append: + tag: append_process_hash_sha1_0bcfc307 field: process.hash.sha1 value: '{{{json.process.process_hash}}}' if: ctx.json?.process?.process_hash != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_hash_18e9b98c field: related.hash value: '{{{json.process.process_hash}}}' if: ctx.json?.process?.process_hash != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_process_process_id_to_jamf_compliance_reporter_log_process_pid_722f5366 field: json.process.process_id target_field: jamf_compliance_reporter.log.process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_1161a51b field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_process_process_name_to_process_name_19b5c16c field: json.process.process_name target_field: process.name ignore_missing: true - convert: + tag: convert_json_process_session_id_to_jamf_compliance_reporter_log_process_session_id_4c2d9502 field: json.process.session_id target_field: jamf_compliance_reporter.log.process.session.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_process_terminal_id_addr_to_jamf_compliance_reporter_log_process_terminal_id_addr_4e0315dd field: json.process.terminal_id.addr target_field: jamf_compliance_reporter.log.process.terminal_id.addr type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_process_terminal_id_ip_address_to_jamf_compliance_reporter_log_process_terminal_id_ip_address_68d5e2c3 field: json.process.terminal_id.ip_address target_field: jamf_compliance_reporter.log.process.terminal_id.ip_address type: ip ignore_missing: true on_failure: - remove: + tag: remove_json_process_terminal_id_ip_address_56c5fe6d field: json.process.terminal_id.ip_address ignore_missing: true - append: + tag: append_error_message_c4abb806 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_related_ip_6572b95d field: related.ip value: '{{{jamf_compliance_reporter.log.process.terminal_id.ip_address}}}' if: ctx.jamf_compliance_reporter?.log?.process?.terminal_id?.ip_address != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_process_terminal_id_port_to_jamf_compliance_reporter_log_process_terminal_id_port_356a52fa field: json.process.terminal_id.port target_field: jamf_compliance_reporter.log.process.terminal_id.port type: long ignore_missing: true on_failure: - append: + tag: append_error_message_b1d00e8d field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_process_terminal_id_type_to_jamf_compliance_reporter_log_process_terminal_id_type_4f680d17 field: json.process.terminal_id.type target_field: jamf_compliance_reporter.log.process.terminal_id.type type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_process_user_id_to_jamf_compliance_reporter_log_process_user_id_57ddfe58 field: json.process.user_id target_field: jamf_compliance_reporter.log.process.user.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_process_user_name_to_jamf_compliance_reporter_log_process_user_name_9da5612b field: json.process.user_name target_field: jamf_compliance_reporter.log.process.user.name ignore_missing: true - append: + tag: append_related_user_8264b549 field: related.user value: '{{{jamf_compliance_reporter.log.process.user.name}}}' if: ctx.jamf_compliance_reporter?.log?.process?.user?.name != null @@ -151,3 +179,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_prohibited_app_blocked.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_prohibited_app_blocked.yml index 1da8afa7087..35e5bbedb91 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_prohibited_app_blocked.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_prohibited_app_blocked.yml @@ -2,262 +2,315 @@ description: Pipeline for Jamf Compliance Reporter prohibited app blocked event logs. processors: - rename: + tag: rename_json_header_action_to_jamf_compliance_reporter_log_header_action_4e424232 field: json.header.action target_field: jamf_compliance_reporter.log.header.action ignore_missing: true - rename: + tag: rename_json_exec_args_args_to_json_args_c2752467 field: json.exec_args.args target_field: json.args ignore_missing: true - rename: + tag: rename_json_exec_args_args_compiled_to_jamf_compliance_reporter_log_exec_args_args_compiled_6e9208ee field: json.exec_args.args_compiled target_field: jamf_compliance_reporter.log.exec_args.args_compiled ignore_missing: true - rename: + tag: rename_json_exec_env_env_PATH_to_jamf_compliance_reporter_log_exec_env_env_path_66b7d43e field: json.exec_env.env.PATH target_field: jamf_compliance_reporter.log.exec_env.env.path ignore_missing: true - rename: + tag: rename_json_exec_env_env_SHELL_to_jamf_compliance_reporter_log_exec_env_env_shell_b8736590 field: json.exec_env.env.SHELL target_field: jamf_compliance_reporter.log.exec_env.env.shell ignore_missing: true - rename: + tag: rename_json_exec_env_env_SSH_AUTH_SOCK_to_jamf_compliance_reporter_log_exec_env_env_ssh_auth_sock_586b6f74 field: json.exec_env.env.SSH_AUTH_SOCK target_field: jamf_compliance_reporter.log.exec_env.env.ssh_auth_sock ignore_missing: true - rename: + tag: rename_json_exec_env_env_TMPDIR_to_jamf_compliance_reporter_log_exec_env_env_tmpdir_630f97d2 field: json.exec_env.env.TMPDIR target_field: jamf_compliance_reporter.log.exec_env.env.tmpdir ignore_missing: true - append: + tag: append_user_name_b215dbf8 field: user.name value: '{{{json.exec_env.env.USER}}}' if: ctx.json?.exec_env?.env?.USER != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_1af882e8 field: related.user value: '{{{json.exec_env.env.USER}}}' if: ctx.json?.exec_env?.env?.USER != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_exec_env_env_XPC_FLAGS_to_jamf_compliance_reporter_log_exec_env_env_xpc_flags_10e3a485 field: json.exec_env.env.XPC_FLAGS target_field: jamf_compliance_reporter.log.exec_env.env.xpc.flags ignore_missing: true - rename: + tag: rename_json_exec_env_env_XPC_SERVICE_NAME_to_jamf_compliance_reporter_log_exec_env_env_xpc_service_name_b71ea807 field: json.exec_env.env.XPC_SERVICE_NAME target_field: jamf_compliance_reporter.log.exec_env.env.xpc.service_name ignore_missing: true - rename: + tag: rename_json_exec_env_env_compiled_to_jamf_compliance_reporter_log_exec_env_env_compiled_42f5d016 field: json.exec_env.env_compiled target_field: jamf_compliance_reporter.log.exec_env.env_compiled ignore_missing: true - rename: + tag: rename_json_identity_cd_hash_to_jamf_compliance_reporter_log_identity_cd_hash_a91be7b4 field: json.identity.cd_hash target_field: jamf_compliance_reporter.log.identity.cd_hash ignore_missing: true - append: + tag: append_related_hash_04fe9d31 field: related.hash value: '{{{jamf_compliance_reporter.log.identity.cd_hash}}}' if: ctx.jamf_compliance_reporter?.log?.identity?.cd_hash != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_identity_signer_id_to_jamf_compliance_reporter_log_identity_signer_id_55e24284 field: json.identity.signer_id target_field: jamf_compliance_reporter.log.identity.signer.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_identity_signer_id_truncated_to_jamf_compliance_reporter_log_identity_signer_id_truncated_7d4200ce field: json.identity.signer_id_truncated target_field: jamf_compliance_reporter.log.identity.signer.id_truncated type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_identity_signer_type_to_jamf_compliance_reporter_log_identity_signer_type_779a9f36 field: json.identity.signer_type target_field: jamf_compliance_reporter.log.identity.signer.type type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_identity_team_id_to_jamf_compliance_reporter_log_identity_team_id_26560692 field: json.identity.team_id target_field: jamf_compliance_reporter.log.identity.team.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_identity_team_id_truncated_to_jamf_compliance_reporter_log_identity_team_id_truncated_20bad62c field: json.identity.team_id_truncated target_field: jamf_compliance_reporter.log.identity.team.id_truncated type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_subject_audit_id_to_process_real_user_id_25b75fe6 field: json.subject.audit_id target_field: process.real_user.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_subject_audit_user_name_to_process_real_user_name_9efe64b1 field: json.subject.audit_user_name target_field: process.real_user.name ignore_missing: true - convert: + tag: convert_json_subject_effective_group_id_to_jamf_compliance_reporter_log_subject_effective_group_id_08ee1ee7 field: json.subject.effective_group_id target_field: jamf_compliance_reporter.log.subject.effective.group.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_subject_effective_group_name_to_jamf_compliance_reporter_log_subject_effective_group_name_c257af46 field: json.subject.effective_group_name target_field: jamf_compliance_reporter.log.subject.effective.group.name ignore_missing: true - convert: + tag: convert_json_subject_effective_user_id_to_user_effective_id_8561a1e7 field: json.subject.effective_user_id target_field: user.effective.id type: string ignore_missing: true ignore_failure: true - set: + tag: set_jamf_compliance_reporter_log_subject_effective_user_id_76b205b4 field: jamf_compliance_reporter.log.subject.effective.user.id copy_from: user.effective.id ignore_failure: true - append: + tag: append_user_name_6426d56f field: user.name value: '{{{json.subject.effective_user_name}}}' if: ctx.json?.subject?.effective_user_name != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_6dba669f field: related.user value: '{{{json.subject.effective_user_name}}}' if: ctx.json?.subject?.effective_user_name != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_subject_effective_user_name_to_user_effective_name_7f204018 field: json.subject.effective_user_name target_field: user.effective.name ignore_missing: true - set: + tag: set_jamf_compliance_reporter_log_subject_effective_user_name_6c475540 field: jamf_compliance_reporter.log.subject.effective.user.name copy_from: user.effective.name ignore_failure: true - convert: + tag: convert_json_subject_group_id_to_user_group_id_6aa8aaab field: json.subject.group_id target_field: user.group.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_subject_group_name_to_user_group_name_4989cf8e field: json.subject.group_name target_field: user.group.name ignore_missing: true - rename: + tag: rename_json_subject_process_hash_to_process_hash_sha1_ec183692 field: json.subject.process_hash target_field: process.hash.sha1 ignore_missing: true - append: + tag: append_related_hash_8db6a16a field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_subject_process_id_to_jamf_compliance_reporter_log_subject_process_pid_c6218d3b field: json.subject.process_id target_field: jamf_compliance_reporter.log.subject.process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_b970a4a6 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_subject_process_information_to_jamf_compliance_reporter_log_subject_process_information_a98a03c5 field: json.subject.process_information target_field: jamf_compliance_reporter.log.subject.process.information ignore_missing: true - rename: + tag: rename_json_subject_process_name_to_process_name_1f8c84e1 field: json.subject.process_name target_field: process.name ignore_missing: true - convert: + tag: convert_json_subject_responsible_process_id_to_jamf_compliance_reporter_log_subject_responsible_process_id_396f9331 field: json.subject.responsible_process_id target_field: jamf_compliance_reporter.log.subject.responsible.process.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_subject_responsible_process_name_to_jamf_compliance_reporter_log_subject_responsible_process_name_f33bae38 field: json.subject.responsible_process_name target_field: jamf_compliance_reporter.log.subject.responsible.process.name ignore_missing: true - convert: + tag: convert_json_subject_session_id_to_jamf_compliance_reporter_log_subject_session_id_f51f0574 field: json.subject.session_id target_field: jamf_compliance_reporter.log.subject.session.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_subject_terminal_id_ip_address_to_json_subject_terminal_id_ip_address_690dae36 field: json.subject.terminal_id.ip_address target_field: json.subject.terminal_id.ip_address type: ip ignore_missing: true on_failure: - remove: + tag: remove_json_subject_terminal_id_ip_address_840b9479 field: json.subject.terminal_id.ip_address ignore_missing: true - append: + tag: append_error_message_12e8ac6b field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_host_ip_e4f236dd field: host.ip value: '{{{json.subject.terminal_id.ip_address}}}' if: ctx.json?.subject?.terminal_id?.ip_address != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_ip_8541d760 field: related.ip value: '{{{json.subject.terminal_id.ip_address}}}' if: ctx.json?.subject?.terminal_id?.ip_address != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_subject_terminal_id_port_to_jamf_compliance_reporter_log_subject_terminal_id_port_301d7d36 field: json.subject.terminal_id.port target_field: jamf_compliance_reporter.log.subject.terminal_id.port type: long ignore_missing: true on_failure: - append: + tag: append_error_message_9d728ff1 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_subject_terminal_id_type_to_jamf_compliance_reporter_log_subject_terminal_id_type_8a893ed5 field: json.subject.terminal_id.type target_field: jamf_compliance_reporter.log.subject.terminal_id.type type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_subject_user_id_to_user_id_19eb600e field: json.subject.user_id target_field: user.id type: string ignore_missing: true ignore_failure: true - append: + tag: append_user_name_11035ae3 field: user.name value: '{{{json.subject.user_name}}}' if: ctx.json?.subject?.user_name != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_acadb613 field: related.user value: '{{{json.subject.user_name}}}' if: ctx.json?.subject?.user_name != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_texts_to_jamf_compliance_reporter_log_texts_e75677b2 field: json.texts target_field: jamf_compliance_reporter.log.texts ignore_missing: true - script: + tag: script_721d76b6 description: Convert Object into Array. lang: painless source: | @@ -275,3 +328,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_signal_event.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_signal_event.yml index e4d468934bd..d4951bd5164 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_signal_event.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_signal_event.yml @@ -2,12 +2,14 @@ description: Pipeline for Jamf Compliance Reporter signal event logs. processors: - convert: + tag: convert_json_signal_event_info_signal_to_jamf_compliance_reporter_log_signal_event_info_signal_6e4a5cde field: json.signal_event_info.signal target_field: jamf_compliance_reporter.log.signal_event_info.signal type: long ignore_missing: true on_failure: - append: + tag: append_error_message_7019edf1 field: error.message value: '{{{_ingest.on_failure_message}}}' on_failure: @@ -17,3 +19,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_unified_log_event.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_unified_log_event.yml index 6ed92eca548..c827484b5ac 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_unified_log_event.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_unified_log_event.yml @@ -2,12 +2,14 @@ description: Pipeline for Jamf Compliance Reporter unified event logs. processors: - convert: + tag: convert_json_event_attributes_activityIdentifier_to_jamf_compliance_reporter_log_event_attributes_activity_identifier_6972ae84 field: json.event_attributes.activityIdentifier target_field: jamf_compliance_reporter.log.event_attributes.activity_identifier type: string ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_event_attributes_backtrace_frames_66890ef3 field: json.event_attributes.backtrace.frames if: ctx.json?.event_attributes?.backtrace?.frames instanceof List processor: @@ -23,6 +25,7 @@ processors: field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_event_attributes_backtrace_frames_f567ad80 field: json.event_attributes.backtrace.frames if: ctx.json?.event_attributes?.backtrace?.frames instanceof List processor: @@ -30,6 +33,7 @@ processors: field: _ingest._value.imageOffset ignore_failure: true - foreach: + tag: foreach_json_event_attributes_backtrace_frames_359aa530 field: json.event_attributes.backtrace.frames if: ctx.json?.event_attributes?.backtrace?.frames instanceof List processor: @@ -38,92 +42,112 @@ processors: target_field: _ingest._value.image_uuid ignore_failure: true - rename: + tag: rename_json_event_attributes_backtrace_frames_to_jamf_compliance_reporter_log_event_attributes_backtrace_frames_429613c2 field: json.event_attributes.backtrace.frames target_field: jamf_compliance_reporter.log.event_attributes.backtrace.frames ignore_missing: true - rename: + tag: rename_json_event_attributes_category_to_jamf_compliance_reporter_log_event_attributes_category_aff19ff2 field: json.event_attributes.category target_field: jamf_compliance_reporter.log.event_attributes.category ignore_missing: true - rename: + tag: rename_json_event_attributes_eventMessage_to_jamf_compliance_reporter_log_event_attributes_event_message_913a1592 field: json.event_attributes.eventMessage target_field: jamf_compliance_reporter.log.event_attributes.event.message ignore_missing: true - rename: + tag: rename_json_event_attributes_eventType_to_jamf_compliance_reporter_log_event_attributes_event_type_05617a1a field: json.event_attributes.eventType target_field: jamf_compliance_reporter.log.event_attributes.event.type ignore_missing: true - rename: + tag: rename_json_event_attributes_formatString_to_jamf_compliance_reporter_log_event_attributes_format_string_c0a32007 field: json.event_attributes.formatString target_field: jamf_compliance_reporter.log.event_attributes.format_string ignore_missing: true - convert: + tag: convert_json_event_attributes_machTimestamp_to_jamf_compliance_reporter_log_event_attributes_mach_timestamp_a2649842 field: json.event_attributes.machTimestamp target_field: jamf_compliance_reporter.log.event_attributes.mach_timestamp type: string ignore_missing: true ignore_failure: true - append: + tag: append_event_type_b4258b86 field: event.type value: 'info' if: ctx.json?.event_attributes?.messageType != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_event_attributes_parentActivityIdentifier_to_jamf_compliance_reporter_log_event_attributes_parent_activity_identifier_8ffc9451 field: json.event_attributes.parentActivityIdentifier target_field: jamf_compliance_reporter.log.event_attributes.parent_activity_identifier type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_event_attributes_processID_to_jamf_compliance_reporter_log_event_attributes_process_id_05fec6b2 field: json.event_attributes.processID target_field: jamf_compliance_reporter.log.event_attributes.process.id type: long ignore_missing: true on_failure: - append: + tag: append_error_message_6bc3ee23 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_event_attributes_processImagePath_to_jamf_compliance_reporter_log_event_attributes_process_image_path_9727bd80 field: json.event_attributes.processImagePath target_field: jamf_compliance_reporter.log.event_attributes.process.image.path ignore_missing: true - rename: + tag: rename_json_event_attributes_processImageUUID_to_jamf_compliance_reporter_log_event_attributes_process_image_uuid_92647cc8 field: json.event_attributes.processImageUUID target_field: jamf_compliance_reporter.log.event_attributes.process.image.uuid ignore_missing: true - rename: + tag: rename_json_event_attributes_senderImagePath_to_jamf_compliance_reporter_log_event_attributes_sender_image_path_1ebcae94 field: json.event_attributes.senderImagePath target_field: jamf_compliance_reporter.log.event_attributes.sender.image.path ignore_missing: true - rename: + tag: rename_json_event_attributes_senderImageUUID_to_jamf_compliance_reporter_log_event_attributes_sender_image_uuid_3231d394 field: json.event_attributes.senderImageUUID target_field: jamf_compliance_reporter.log.event_attributes.sender.image.uuid ignore_missing: true - convert: + tag: convert_json_event_attributes_senderProgramCounter_to_jamf_compliance_reporter_log_event_attributes_sender_program_counter_2b7b082b field: json.event_attributes.senderProgramCounter target_field: jamf_compliance_reporter.log.event_attributes.sender.program_counter type: long ignore_missing: true on_failure: - append: + tag: append_error_message_810b817c field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_event_attributes_source_to_jamf_compliance_reporter_log_event_attributes_source_72ef1212 field: json.event_attributes.source target_field: jamf_compliance_reporter.log.event_attributes.source ignore_missing: true - rename: + tag: rename_json_event_attributes_subsystem_to_jamf_compliance_reporter_log_event_attributes_subsystem_d3322ff2 field: json.event_attributes.subsystem target_field: jamf_compliance_reporter.log.event_attributes.subsystem ignore_missing: true - convert: + tag: convert_json_event_attributes_threadID_to_jamf_compliance_reporter_log_event_attributes_thread_id_f71be0ec field: json.event_attributes.threadID target_field: jamf_compliance_reporter.log.event_attributes.thread_id type: string ignore_missing: true ignore_failure: true - date: + tag: date_json_event_attributes_timestamp_to_jamf_compliance_reporter_log_event_attributes_timestamp_35bc5640 field: json.event_attributes.timestamp target_field: jamf_compliance_reporter.log.event_attributes.timestamp if: ctx.json?.event_attributes?.timestamp != null && ctx.json.event_attributes.timestamp != 0 @@ -131,13 +155,16 @@ processors: - yyyy-MM-dd HH:mm:ss.SSSSSSZ on_failure: - append: + tag: append_error_message_18db0b33 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_event_attributes_timezoneName_to_jamf_compliance_reporter_log_event_attributes_timezone_name_71b67557 field: json.event_attributes.timezoneName target_field: jamf_compliance_reporter.log.event_attributes.timezone_name ignore_missing: true - convert: + tag: convert_json_event_attributes_traceID_to_jamf_compliance_reporter_log_event_attributes_trace_id_ddce6ab0 field: json.event_attributes.traceID target_field: jamf_compliance_reporter.log.event_attributes.trace_id type: string @@ -150,3 +177,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_xprotect_definitions_version_info.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_xprotect_definitions_version_info.yml index d079ac76d21..25866748503 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_xprotect_definitions_version_info.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_xprotect_definitions_version_info.yml @@ -2,26 +2,32 @@ description: Pipeline for Jamf Compliance Reporter xprotect definitions version info event logs. processors: - rename: + tag: rename_json_event_attributes_BuildAliasOf_to_jamf_compliance_reporter_log_event_attributes_build_alias_of_536b64ca field: json.event_attributes.BuildAliasOf target_field: jamf_compliance_reporter.log.event_attributes.build_alias_of ignore_missing: true - rename: + tag: rename_json_event_attributes_BuildVersion_to_jamf_compliance_reporter_log_event_attributes_build_version_69ed7cb3 field: json.event_attributes.BuildVersion target_field: jamf_compliance_reporter.log.event_attributes.build_version ignore_missing: true - rename: + tag: rename_json_event_attributes_CFBundleShortVersionString_to_jamf_compliance_reporter_log_event_attributes_cf_bundle_short_version_string_4564cdb8 field: json.event_attributes.CFBundleShortVersionString target_field: jamf_compliance_reporter.log.event_attributes.cf_bundle_short_version_string ignore_missing: true - rename: + tag: rename_json_event_attributes_CFBundleVersion_to_jamf_compliance_reporter_log_event_attributes_cf_bundle_version_6c116330 field: json.event_attributes.CFBundleVersion target_field: jamf_compliance_reporter.log.event_attributes.cf_bundle_version ignore_missing: true - rename: + tag: rename_json_event_attributes_ProjectName_to_jamf_compliance_reporter_log_event_attributes_project_name_43e5ffb9 field: json.event_attributes.ProjectName target_field: jamf_compliance_reporter.log.event_attributes.project_name ignore_missing: true - rename: + tag: rename_json_event_attributes_SourceVersion_to_jamf_compliance_reporter_log_event_attributes_source_version_ba02e8f3 field: json.event_attributes.SourceVersion target_field: jamf_compliance_reporter.log.event_attributes.source_version ignore_missing: true @@ -32,3 +38,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_xprotect_event_log.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_xprotect_event_log.yml index 720d9548df3..d4ae6ad8c87 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_xprotect_event_log.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/pipeline_xprotect_event_log.yml @@ -2,6 +2,7 @@ description: Pipeline for Jamf Compliance Reporter xprotect event logs. processors: - foreach: + tag: foreach_json_event_attributes_e51b5751 field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -11,12 +12,14 @@ processors: allow_duplicates: false ignore_failure: true - convert: + tag: convert_jamf_compliance_reporter_log_event_attributes_activity_identifier_to_jamf_compliance_reporter_log_event_attributes_activity_identifier_48a8a699 field: jamf_compliance_reporter.log.event_attributes.activity_identifier target_field: jamf_compliance_reporter.log.event_attributes.activity_identifier type: string ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_event_attributes_2caad3b8 field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -29,15 +32,18 @@ processors: allow_duplicates: false ignore_failure: true - convert: + tag: convert_jamf_compliance_reporter_log_event_attributes_backtrace_frames_image_offset_to_jamf_compliance_reporter_log_event_attributes_backtrace_frames_image_offset_6e9ddd24 field: jamf_compliance_reporter.log.event_attributes.backtrace.frames.image_offset target_field: jamf_compliance_reporter.log.event_attributes.backtrace.frames.image_offset type: long ignore_missing: true on_failure: - append: + tag: append_error_message_2e052947 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_event_attributes_ccb965d8 field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -50,6 +56,7 @@ processors: allow_duplicates: false ignore_failure: true - foreach: + tag: foreach_json_event_attributes_a6a3a796 field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -59,6 +66,7 @@ processors: allow_duplicates: false ignore_failure: true - foreach: + tag: foreach_json_event_attributes_0bd5aec0 field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -68,6 +76,7 @@ processors: allow_duplicates: false ignore_failure: true - foreach: + tag: foreach_json_event_attributes_ae37590e field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -77,6 +86,7 @@ processors: allow_duplicates: false ignore_failure: true - foreach: + tag: foreach_json_event_attributes_90e4cb29 field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -86,6 +96,7 @@ processors: allow_duplicates: false ignore_failure: true - foreach: + tag: foreach_json_event_attributes_2d068064 field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -101,6 +112,7 @@ processors: field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_event_attributes_148e61eb field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -110,6 +122,7 @@ processors: allow_duplicates: false ignore_failure: true - foreach: + tag: foreach_json_event_attributes_ba4435d2 field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -119,12 +132,14 @@ processors: allow_duplicates: false ignore_failure: true - convert: + tag: convert_jamf_compliance_reporter_log_event_attributes_parent_activity_identifier_to_jamf_compliance_reporter_log_event_attributes_parent_activity_identifier_49602cbb field: jamf_compliance_reporter.log.event_attributes.parent_activity_identifier target_field: jamf_compliance_reporter.log.event_attributes.parent_activity_identifier type: string ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_event_attributes_d2b1c2a0 field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -134,15 +149,18 @@ processors: allow_duplicates: false ignore_failure: true - convert: + tag: convert_jamf_compliance_reporter_log_event_attributes_process_id_to_jamf_compliance_reporter_log_event_attributes_process_id_67a8ecd8 field: jamf_compliance_reporter.log.event_attributes.process.id target_field: jamf_compliance_reporter.log.event_attributes.process.id type: long ignore_missing: true on_failure: - append: + tag: append_error_message_361d2267 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_event_attributes_8cc6180a field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -152,6 +170,7 @@ processors: allow_duplicates: false ignore_failure: true - foreach: + tag: foreach_json_event_attributes_5dfcf6ea field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -161,6 +180,7 @@ processors: allow_duplicates: false ignore_failure: true - foreach: + tag: foreach_json_event_attributes_e02cc404 field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -170,6 +190,7 @@ processors: allow_duplicates: false ignore_failure: true - foreach: + tag: foreach_json_event_attributes_3830c48c field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -179,6 +200,7 @@ processors: allow_duplicates: false ignore_failure: true - foreach: + tag: foreach_json_event_attributes_0cb94acf field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -188,15 +210,18 @@ processors: allow_duplicates: false ignore_failure: true - convert: + tag: convert_jamf_compliance_reporter_log_event_attributes_sender_program_counter_to_jamf_compliance_reporter_log_event_attributes_sender_program_counter_c8eadbf0 field: jamf_compliance_reporter.log.event_attributes.sender.program_counter target_field: jamf_compliance_reporter.log.event_attributes.sender.program_counter type: long ignore_missing: true on_failure: - append: + tag: append_error_message_8526b69b field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_event_attributes_732d7cc8 field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -206,6 +231,7 @@ processors: allow_duplicates: false ignore_failure: true - foreach: + tag: foreach_json_event_attributes_996ed252 field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -215,6 +241,7 @@ processors: allow_duplicates: false ignore_failure: true - foreach: + tag: foreach_json_event_attributes_a5d149ad field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -224,12 +251,14 @@ processors: allow_duplicates: false ignore_failure: true - convert: + tag: convert_jamf_compliance_reporter_log_event_attributes_thread_id_to_jamf_compliance_reporter_log_event_attributes_thread_id_97d92773 field: jamf_compliance_reporter.log.event_attributes.thread_id target_field: jamf_compliance_reporter.log.event_attributes.thread_id type: string ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_event_attributes_bfe87000 field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -246,6 +275,7 @@ processors: field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_event_attributes_71b8fd1a field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -255,6 +285,7 @@ processors: allow_duplicates: false ignore_failure: true - foreach: + tag: foreach_json_event_attributes_aabcb896 field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -264,6 +295,7 @@ processors: allow_duplicates: false ignore_failure: true - foreach: + tag: foreach_json_event_attributes_f9d45d9f field: json.event_attributes if: ctx.json?.event_attributes instanceof List processor: @@ -273,6 +305,7 @@ processors: allow_duplicates: false ignore_failure: true - convert: + tag: convert_jamf_compliance_reporter_log_event_attributes_trace_id_to_jamf_compliance_reporter_log_event_attributes_trace_id_a4db3577 field: jamf_compliance_reporter.log.event_attributes.trace_id target_field: jamf_compliance_reporter.log.event_attributes.trace_id type: string @@ -285,3 +318,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_compliance_reporter/manifest.yml b/packages/jamf_compliance_reporter/manifest.yml index c9622ee09bf..23ad1b70294 100644 --- a/packages/jamf_compliance_reporter/manifest.yml +++ b/packages/jamf_compliance_reporter/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: jamf_compliance_reporter title: Jamf Compliance Reporter -version: "1.16.0" +version: "1.16.1" description: Collect logs from Jamf Compliance Reporter with Elastic Agent. type: integration categories: diff --git a/packages/jamf_pro/changelog.yml b/packages/jamf_pro/changelog.yml index 259ff0ca213..879cb1dcf5b 100644 --- a/packages/jamf_pro/changelog.yml +++ b/packages/jamf_pro/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.0.1" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "2.0.0" changes: - description: Add latest transform to the inventory data stream. diff --git a/packages/jamf_pro/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/jamf_pro/data_stream/events/elasticsearch/ingest_pipeline/default.yml index b3e3201044d..376bd8196f4 100644 --- a/packages/jamf_pro/data_stream/events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/jamf_pro/data_stream/events/elasticsearch/ingest_pipeline/default.yml @@ -3,22 +3,25 @@ description: Pipeline for processing sample logs processors: - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - rename: + tag: rename_json_to_jamf_pro_events_127fc70c field: json target_field: jamf_pro.events # all valid events will now have jamf_pro.events.{event,webhook} - convert: + tag: convert_jamf_pro_events_event_management_id_d79c3147 field: jamf_pro.events.event.management_id type: string if: > ctx.jamf_pro.events.event?.management_id != null && !(ctx.jamf_pro.events.event.management_id instanceof String) - - rename: + tag: rename_jamf_pro_events_event_computer_to_jamf_pro_events_event_is_computer_76427444 field: jamf_pro.events.event.computer target_field: jamf_pro.events.event.is_computer if: ctx.jamf_pro.events.event.computer instanceof boolean @@ -57,17 +60,19 @@ processors: ctx.jamf_pro.events = keysToSnakeCase(ctx.jamf_pro.events); } + ############## # Timestamps # ############## - - date: + tag: date_jamf_pro_events_webhook_event_timestamp_to_jamf_pro_events_webhook_event_timestamp_31d0fb7a field: jamf_pro.events.webhook.event_timestamp target_field: jamf_pro.events.webhook.event_timestamp formats: - UNIX_MS - date: + tag: date_jamf_pro_events_event_last_update_to_jamf_pro_events_event_last_update_4f3eb8b0 field: jamf_pro.events.event.last_update target_field: jamf_pro.events.event.last_update if: ctx.jamf_pro.events.event.last_update != null @@ -75,64 +80,73 @@ processors: - UNIX_MS - set: + tag: set_host_entity_lifecycle_last_activity_1ddc1893 field: host.entity.lifecycle.last_activity copy_from: jamf_pro.events.webhook.event_timestamp ignore_empty_value: true + ############## # IP src # ############## - - set: + tag: set_host_ip_4499d624 if: "ctx.jamf_pro.events.event.ip_address != null && ctx.jamf_pro.events.event.ip_address != ''" field: host.ip - value: + value: - '{{{jamf_pro.events.event.ip_address}}}' ignore_empty_value: true - set: + tag: set_host_ip_6dd53242 if: > ctx.jamf_pro.events.event.computer?.reported_ip_address != null && ctx.jamf_pro.events.event.computer.reported_ip_address != '' field: host.ip - value: + value: - "{{{jamf_pro.events.event.computer.reported_ip_address}}}" ignore_empty_value: true + ############## # User src # ############## - - set: + tag: set_user_name_6ce75428 field: user.name copy_from: jamf_pro.events.event.username ignore_empty_value: true - set: + tag: set_user_name_cb89921b field: user.name copy_from: jamf_pro.events.event.computer.username ignore_empty_value: true - set: + tag: set_user_email_8e078ecb field: user.email copy_from: jamf_pro.events.event.computer.email_address ignore_empty_value: true - set: + tag: set_user_email_0f3c2b38 field: user.email copy_from: jamf_pro.events.event.email_address ignore_empty_value: true + ############## # OS src # ############## - - set: + tag: set_host_os_version_524e1d1a if: ctx.jamf_pro.events.event.os_version != null field: host.os.version copy_from: jamf_pro.events.event.os_version - set: + tag: set_host_os_version_9e9967cd if: ctx.jamf_pro.events.event.computer?.os_version != null field: host.os.version copy_from: jamf_pro.events.event.computer.os_version @@ -141,43 +155,50 @@ processors: ############## # ECS compat # ############## - - set: + tag: set_host_name_604445a4 field: host.name copy_from: jamf_pro.events.event.computer.device_name ignore_empty_value: true - set: + tag: set_host_id_7ce14fbc field: host.id copy_from: jamf_pro.events.event.computer.udid ignore_empty_value: true - set: + tag: set_host_address_8760bf68 field: host.address copy_from: host.ip ignore_empty_value: true - geoip: + tag: geoip_host_ip_to_host_geo_e7ddacb9 if: ctx.host?.ip != null && ctx.host.ip != '' field: host.ip target_field: host.geo ignore_missing: true - append: + tag: append_related_user_36df74b3 field: related.user value: "{{{user.name}}}" if: ctx.user?.name != null && ctx.user.name != "" - append: + tag: append_related_user_968086e6 field: related.user value: "{{{user.email}}}" if: ctx.user?.email != null && ctx.user.email != "" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_action_17ff02f3 field: event.action copy_from: jamf_pro.events.webhook.webhook_event ignore_empty_value: true @@ -189,28 +210,28 @@ processors: description: Set event.category and event.type from a table keyed on the Jamf webhook event (event.action). params: actions: - ComputerAdded: { category: [host], type: [change] } - ComputerCheckIn: { category: [host], type: [] } - ComputerInventoryCompleted: { category: [host], type: [] } - ComputerPushCapabilityChanged: { category: [host], type: [change] } - MobileDeviceEnrolled: { category: [host], type: [change, start] } - MobileDeviceUnEnrolled: { category: [host], type: [change, end] } - MobileDeviceCheckIn: { category: [host], type: [] } - MobileDeviceInventoryCompleted: { category: [host], type: [] } - MobileDeviceCommandCompleted: { category: [host], type: [change, end] } - MobileDevicePushSent: { category: [host], type: [] } - DeviceAddedToDEP: { category: [host], type: [change] } - PushSent: { category: [host], type: [] } - SCEPChallenge: { category: [host, authentication], type: [start] } - SmartGroupComputerMembershipChange: { category: [host, iam], type: [change, group] } - SmartGroupMobileDeviceMembershipChange: { category: [host, iam], type: [change, group] } - JSSStartup: { category: [host], type: [start] } - JSSShutdown: { category: [host], type: [end] } - ComputerPolicyFinished: { category: [configuration], type: [change] } - ComputerPatchPolicyCompleted: { category: [configuration, package], type: [change] } - PatchSoftwareTitleUpdated: { category: [configuration, package], type: [change] } - RestAPIOperation: { category: [api], type: [change, admin] } - SmartGroupUserMembershipChange: { category: [iam], type: [change, group] } + ComputerAdded: {category: [host], type: [change]} + ComputerCheckIn: {category: [host], type: []} + ComputerInventoryCompleted: {category: [host], type: []} + ComputerPushCapabilityChanged: {category: [host], type: [change]} + MobileDeviceEnrolled: {category: [host], type: [change, start]} + MobileDeviceUnEnrolled: {category: [host], type: [change, end]} + MobileDeviceCheckIn: {category: [host], type: []} + MobileDeviceInventoryCompleted: {category: [host], type: []} + MobileDeviceCommandCompleted: {category: [host], type: [change, end]} + MobileDevicePushSent: {category: [host], type: []} + DeviceAddedToDEP: {category: [host], type: [change]} + PushSent: {category: [host], type: []} + SCEPChallenge: {category: [host, authentication], type: [start]} + SmartGroupComputerMembershipChange: {category: [host, iam], type: [change, group]} + SmartGroupMobileDeviceMembershipChange: {category: [host, iam], type: [change, group]} + JSSStartup: {category: [host], type: [start]} + JSSShutdown: {category: [host], type: [end]} + ComputerPolicyFinished: {category: [configuration], type: [change]} + ComputerPatchPolicyCompleted: {category: [configuration, package], type: [change]} + PatchSoftwareTitleUpdated: {category: [configuration, package], type: [change]} + RestAPIOperation: {category: [api], type: [change, admin]} + SmartGroupUserMembershipChange: {category: [iam], type: [change, group]} source: |- def action = ctx.event?.action; if (action == null) { @@ -236,7 +257,6 @@ processors: if: >- ctx.jamf_pro?.events?.event?.successful == true || ctx.jamf_pro?.events?.event?.operation_successful == true - - set: field: event.outcome tag: set_event_outcome_failure @@ -244,8 +264,8 @@ processors: if: >- ctx.jamf_pro?.events?.event?.successful == false || ctx.jamf_pro?.events?.event?.operation_successful == false - - convert: + tag: convert_jamf_pro_events_event_computer_ip_address_to_source_ip_f0398d65 target_field: source.ip type: ip field: jamf_pro.events.event.computer.ip_address diff --git a/packages/jamf_pro/data_stream/inventory/elasticsearch/ingest_pipeline/default.yml b/packages/jamf_pro/data_stream/inventory/elasticsearch/ingest_pipeline/default.yml index a80f000c297..60ced19f147 100644 --- a/packages/jamf_pro/data_stream/inventory/elasticsearch/ingest_pipeline/default.yml +++ b/packages/jamf_pro/data_stream/inventory/elasticsearch/ingest_pipeline/default.yml @@ -3,10 +3,12 @@ description: Process inventory data from the Jamf Pro API processors: - rename: + tag: rename_message_to_jamf_pro_inventory_d5de13ec field: message target_field: jamf_pro.inventory - script: + tag: script_a3eb5add description: Drops null/empty values recursively. lang: painless source: | @@ -59,41 +61,49 @@ processors: } - fingerprint: + tag: fingerprint_a6dde3e1 fields: - jamf_pro.inventory.udid - jamf_pro.inventory.general.report_date target_field: "_id" - geoip: + tag: geoip_jamf_pro_inventory_general_last_ip_address_to_jamf_pro_inventory_general_last_ip_address_geo_51eed0b8 field: jamf_pro.inventory.general.last_ip_address target_field: jamf_pro.inventory.general.last_ip_address_geo ignore_missing: true - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' + ############## # ECS compat # ############## - - append: + tag: append_host_ip_036ddbf2 if: "ctx.jamf_pro.inventory.general != null && ctx.jamf_pro.inventory.general.last_ip_address != ''" field: host.ip value: '{{{jamf_pro.inventory.general.last_ip_address}}}' ignore_failure: true - set: + tag: set_host_id_d9f48262 field: host.id copy_from: jamf_pro.inventory.udid ignore_empty_value: true - set: + tag: set_host_name_3e068ef3 field: host.name copy_from: jamf_pro.inventory.general.name ignore_empty_value: true - set: + tag: set_host_address_8760bf68 field: host.address copy_from: host.ip ignore_empty_value: true - append: + tag: append_host_mac_bcfc0ce8 if: "ctx.jamf_pro.inventory.hardware != null && ctx.jamf_pro.inventory.hardware.mac_address != ''" field: host.mac value: '{{{jamf_pro.inventory.hardware.mac_address}}}' @@ -106,9 +116,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_host_mac_27cf896f field: host.mac ignore_missing: true - append: + tag: append_error_message_1031db47 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - uppercase: @@ -118,25 +130,31 @@ processors: if: ctx.host?.mac != '' on_failure: - remove: + tag: remove_host_mac_762f8874 field: host.mac ignore_missing: true - append: + tag: append_error_message_6b5e084c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_host_geo_9f1f0bd4 field: host.geo copy_from: jamf_pro.inventory.general.last_ip_address_geo ignore_empty_value: true ignore_failure: true - set: + tag: set_host_entity_attributes_managed_bfbdd343 field: host.entity.attributes.managed copy_from: jamf_pro.inventory.general.remote_management.managed ignore_empty_value: true - set: + tag: set_host_entity_lifecycle_last_activity_6e6e376f field: host.entity.lifecycle.last_activity copy_from: jamf_pro.inventory.general.last_contact_time ignore_empty_value: true - set: + tag: set_host_os_name_382c3eba field: host.os.name copy_from: jamf_pro.inventory.operating_system.name ignore_empty_value: true @@ -213,26 +231,32 @@ processors: ctx.host.os.full = name; } - set: + tag: set_host_os_version_167e76e0 field: host.os.version copy_from: jamf_pro.inventory.operating_system.version ignore_empty_value: true - set: + tag: set_host_os_platform_f077c83b field: host.os.platform copy_from: jamf_pro.inventory.general.platform ignore_empty_value: true - set: + tag: set_user_name_78d20b57 field: user.name copy_from: jamf_pro.inventory.user_and_location.name ignore_empty_value: true - set: + tag: set_user_email_41e85901 field: user.email copy_from: jamf_pro.inventory.user_and_location.email ignore_empty_value: true - set: + tag: set_user_full_name_4beacbed field: user.full_name copy_from: jamf_pro.inventory.user_and_location.realname ignore_empty_value: true - foreach: + tag: foreach_jamf_pro_inventory_group_memberships_b5a5a3a2 field: jamf_pro.inventory.group_memberships if: ctx.jamf_pro.inventory?.group_memberships != null processor: @@ -243,6 +267,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_jamf_pro_inventory_group_memberships_3a061922 field: jamf_pro.inventory.group_memberships if: ctx.jamf_pro.inventory?.group_memberships != null processor: @@ -253,29 +278,34 @@ processors: ignore_missing: true ignore_failure: true - append: + tag: append_related_user_93408443 field: related.user value: '{{{user.email}}}' if: ctx.user?.email != null && ctx.user.email != "" allow_duplicates: false - set: + tag: set_related_ip_3008f8de field: related.ip copy_from: host.ip ignore_empty_value: true + #################### # Asset definition # #################### - - set: + tag: set_event_kind_18c57bf4 field: event.kind value: asset - append: + tag: append_event_type_7a1a4ee6 field: event.type value: user if: ctx.jamf_pro.inventory.user_and_location != null allow_duplicates: false - append: + tag: append_event_type_2e1f55b2 field: event.type value: info if: ctx.jamf_pro.inventory.hardware != null diff --git a/packages/jamf_pro/manifest.yml b/packages/jamf_pro/manifest.yml index cb8bba32404..23337258f98 100644 --- a/packages/jamf_pro/manifest.yml +++ b/packages/jamf_pro/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: jamf_pro title: "Jamf Pro" -version: "2.0.0" +version: "2.0.1" source: license: "Elastic-2.0" description: "Collect logs and inventory data from Jamf Pro with Elastic Agent" diff --git a/packages/jamf_protect/changelog.yml b/packages/jamf_protect/changelog.yml index 6a25940b143..be9396ff7d4 100644 --- a/packages/jamf_protect/changelog.yml +++ b/packages/jamf_protect/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.4.1" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "3.4.0" changes: - description: Set 'elastic' owner type. diff --git a/packages/jamf_protect/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml b/packages/jamf_protect/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml index 0b7f8709e8b..fa86309b136 100644 --- a/packages/jamf_protect/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml +++ b/packages/jamf_protect/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml @@ -2,22 +2,27 @@ description: Pipeline for parsing Jamf Protect Alert and Unified Logging logs processors: - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - rename: + tag: rename_json_to_jamf_protect_alerts_673b9841 field: json target_field: jamf_protect.alerts if: ctx.json != null - json: + tag: json_event_original_to_jamf_protect_alerts_f25b40ec field: event.original target_field: jamf_protect.alerts if: ctx.event?.original != null && ctx.jamf_protect?.alerts == null - script: + tag: script_e808a1cd # This should almost never be needed and will not allow preservation of events in the case # of ingest pipeline errors unless preserve_original_event was already set, in which case # it would already have been retained. @@ -28,41 +33,50 @@ processors: ctx.event = ctx.event ?: [:]; ctx.event.original = Json.dump(ctx.jamf_protect.alerts); -####################### -## ECS Event Mapping ## -####################### + ####################### + ## ECS Event Mapping ## + ####################### - set: + tag: set_event_kind_39295792 field: event.kind value: alert - set: + tag: set_event_provider_7e881320 field: event.provider value: Jamf Protect - date: + tag: date_jamf_protect_alerts_input_match_event_timestamp_to_event_start_3fedf98b field: jamf_protect.alerts.input.match.event.timestamp target_field: event.start - formats: + formats: - UNIX if: ctx.jamf_protect?.alerts?.input?.match?.event?.timestamp != null - append: + tag: append_event_category_3fa422a0 field: event.category value: host - append: + tag: append_event_category_710d494c field: event.category value: malware if: ctx.jamf_protect?.alerts?.input?.eventType == 'GPThreatMatchExecEvent' - append: + tag: append_event_category_95ad9341 field: event.category value: process if: ctx.jamf_protect?.alerts?.input?.eventType == 'GPProcessEvent' - append: + tag: append_event_category_6092eab4 field: event.category value: file if: ctx.jamf_protect?.alerts?.input?.eventType == 'GPFSEvent' - append: + tag: append_event_category_074137cb field: event.category value: file if: ctx.jamf_protect?.alerts?.input?.eventType == 'GPDownloadEvent' - script: + tag: script_e17bb3af if: >- ctx.jamf_protect?.alerts?.input?.match?.facts instanceof List && ctx.jamf_protect.alerts.input.match.facts.length != 0 && @@ -74,6 +88,7 @@ processors: } ctx.rule.name = ctx.jamf_protect.alerts.input.match.facts[0].name - script: + tag: script_b91cc6c8 if: >- ctx.jamf_protect?.alerts?.input?.match?.facts instanceof List && ctx.jamf_protect.alerts.input.match.facts.length != 0 && @@ -85,6 +100,7 @@ processors: } ctx.rule.description = ctx.jamf_protect.alerts.input.match.facts[0].human - script: + tag: script_c801a44a description: Populates event.action with the name of the event lang: painless if: >- @@ -95,6 +111,7 @@ processors: ctx.event.action = ctx.jamf_protect.alerts.input.match.facts[0].name; ignore_failure: true - script: + tag: script_54f25306 description: Populates event.reason with the name of the event lang: painless if: >- @@ -105,6 +122,7 @@ processors: ctx.event.reason = ctx.jamf_protect.alerts.input.match.facts[0].human; ignore_failure: true - script: + tag: script_e319e75b description: Populates event.tags with tags from the event lang: painless if: >- @@ -123,60 +141,71 @@ processors: ctx.tags = tags; ignore_failure: true - rename: + tag: rename_jamf_protect_alerts_input_match_uuid_to_event_id_aafa2940 field: jamf_protect.alerts.input.match.uuid target_field: event.id ignore_missing: true if: ctx.jamf_protect?.alerts?.input?.match?.uuid != null - rename: + tag: rename_jamf_protect_alerts_input_match_severity_to_event_severity_7a73efba field: jamf_protect.alerts.input.match.severity target_field: event.severity ignore_missing: true if: ctx.jamf_protect?.alerts?.input?.match?.severity != null - set: + tag: set_event_outcome_6069b54e field: event.outcome value: failure if: ctx.jamf_protect?.alerts?.input?.match?.actions[0]?.name == "Prevented" - set: + tag: set_event_outcome_8119fc76 field: event.outcome value: success if: ctx.jamf_protect?.alerts?.input?.match?.actions[0]?.name == "Report" - set: + tag: set_event_outcome_fcc5ce88 field: event.outcome value: unknown if: ctx.jamf_protect?.alerts?.input?.match?.actions[0]?.name == null -####################### -## ECS Host Mapping ## -####################### + ####################### + ## ECS Host Mapping ## + ####################### - rename: + tag: rename_jamf_protect_alerts_input_host_hostname_to_host_hostname_9ef39274 field: jamf_protect.alerts.input.host.hostname target_field: host.hostname ignore_missing: true if: ctx.jamf_protect?.alerts?.input?.host?.hostname != null - rename: + tag: rename_jamf_protect_alerts_input_host_provisioningUDID_to_host_id_145e70c0 field: jamf_protect.alerts.input.host.provisioningUDID target_field: host.id ignore_missing: true if: ctx.jamf_protect?.alerts?.input?.host?.provisioningUDID != null - rename: + tag: rename_jamf_protect_alerts_input_host_ips_to_host_ip_453ddc96 field: jamf_protect.alerts.input.host.ips target_field: host.ip ignore_missing: true if: ctx.jamf_protect?.alerts?.input?.host?.ips != null && ctx.jamf_protect?.alerts?.input?.host?.ips != "" - rename: + tag: rename_jamf_protect_alerts_input_host_os_to_host_os_full_7815df94 field: jamf_protect.alerts.input.host.os target_field: host.os.full if: ctx.jamf_protect?.alerts?.input?.host?.os != null ignore_missing: true - set: + tag: set_host_os_family_9d799ed3 field: host.os.family value: macos -######################## -## ECS Source Mapping ## -######################## + ######################## + ## ECS Source Mapping ## + ######################## - script: + tag: script_91332bee description: Script processor to capture user names from related.users array lang: painless source: | @@ -194,31 +223,37 @@ processors: } } -######################## -# ECS File Mapping ## -######################## + ######################## + # ECS File Mapping ## + ######################## - append: + tag: append_event_type_3eec881b field: event.type value: creation if: ctx.jamf_protect?.alerts?.input?.eventType == 'GPFSEvent' && ctx.jamf_protect?.alerts?.input?.match?.event?.type == 0 - append: + tag: append_event_type_0fc7c535 field: event.type value: deletion if: ctx.jamf_protect?.alerts?.input?.eventType == 'GPFSEvent' && ctx.jamf_protect?.alerts?.input?.match?.event?.type == 1 - append: + tag: append_event_type_c7489f41 field: event.type value: change if: ctx.jamf_protect?.alerts?.input?.eventType == 'GPFSEvent' && ctx.jamf_protect?.alerts?.input?.match?.event?.type == 3 - append: + tag: append_event_type_844c6f30 field: event.type value: change if: ctx.jamf_protect?.alerts?.input?.eventType == 'GPFSEvent' && ctx.jamf_protect?.alerts?.input?.match?.event?.type == 4 - append: + tag: append_event_type_a196cbfa field: event.type value: creation if: ctx.jamf_protect?.alerts?.input?.eventType == 'GPFSEvent' && ctx.jamf_protect?.alerts?.input?.match?.event?.type == 7 - script: + tag: script_15c65249 description: One scripting processor to capture all the related file information for the related file. lang: painless if: ctx.jamf_protect?.alerts?.input?.related?.files != null && ctx.jamf_protect?.alerts?.input?.related?.files.size() > 0 @@ -245,19 +280,22 @@ processors: ctx.file.code_signature.team_id = file.signingInfo?.teamid; } -###################### -## ECS Process Mapping ## -###################### + ###################### + ## ECS Process Mapping ## + ###################### - append: + tag: append_event_type_2b3de0c7 field: event.type value: - start if: ctx.jamf_protect?.alerts?.input?.eventType == 'GPProcessEvent' && ctx.jamf_protect?.alerts?.input?.match?.event?.type == 1 - append: + tag: append_event_type_b636d97d field: event.type value: end if: ctx.jamf_protect?.alerts?.input?.eventType == 'GPProcessEvent' && ctx.jamf_protect?.alerts?.input?.match?.event?.type == 2 - script: + tag: script_85a08675 description: One scripting processor to capture all the related process information for the current process. lang: painless if: ctx.jamf_protect?.alerts?.input?.related?.processes != null && ctx.jamf_protect?.alerts?.input?.related?.processes.size() > 0 @@ -392,10 +430,11 @@ processors: } } -###################### -## ECS Group Mapping ## -###################### + ###################### + ## ECS Group Mapping ## + ###################### - script: + tag: script_7a636f11 description: One scripting processor to capture all the related group information for the event. lang: painless source: > @@ -408,27 +447,31 @@ processors: ctx.group.id = group.gid.toString(); } -######################## -# ECS Threat Mapping ## -######################## + ######################## + # ECS Threat Mapping ## + ######################## - set: + tag: set_threat_framework_dd0ef9f4 field: threat.framework value: "MITRE ATT&CK" if: ctx.jamf_protect?.alerts?.input?.match?.facts[0].tags.contains("MITREattack") - set: + tag: set_threat_software_platforms_310c4a3d field: threat.software.platforms value: [macOS] if: ctx.jamf_protect?.alerts?.input?.match?.facts[0].tags.contains("MITREattack") -######################## -# ECS Related Mapping ## -######################## + ######################## + # ECS Related Mapping ## + ######################## - append: + tag: append_related_hosts_ca1f0453 field: related.hosts value: "{{{ host.name }}}" if: ctx.host?.name != null allow_duplicates: false - append: + tag: append_related_user_981c7fa7 field: related.user value: "{{{ user.name }}}" if: ctx.user?.name != null @@ -439,50 +482,56 @@ processors: # if: ctx.host?.ip != null # allow_duplicates: false - foreach: + tag: foreach_host_ip_232e838f field: host.ip if: ctx.host?.ip instanceof List processor: - append: + append: field: related.ip value: '{{{_ingest._value}}}' allow_duplicates: false - append: + tag: append_related_hash_3777cc13 field: related.hash value: "{{{ file.hash.sha1 }}}" if: ctx.file?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_cdcd36b7 field: related.hash value: "{{{ file.hash.sha256 }}}" if: ctx.file?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_e3395a5f field: related.hash value: "{{{ process.hash.sha1 }}}" if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_412c6cb3 field: related.hash value: "{{{ process.hash.sha256 }}}" if: ctx.process?.hash?.sha256 != null allow_duplicates: false - -####################### -## ECS pipelines ## -####################### + ####################### + ## ECS pipelines ## + ####################### - pipeline: + tag: pipeline_49b9aa36 name: '{{ IngestPipeline "gpusbevent" }}' if: ctx.jamf_protect?.alerts?.input?.eventType == "GPUSBEvent" - pipeline: + tag: pipeline_7380978e name: '{{ IngestPipeline "gpunifiedlogevent" }}' if: ctx.jamf_protect?.alerts?.input?.eventType == "GPUnifiedLogEvent" - -############# -## Cleanup ## -############# + ############# + ## Cleanup ## + ############# - remove: + tag: remove_21ac4b10 field: - jamf_protect.alerts - jamf_protect @@ -490,6 +539,7 @@ processors: - json ignore_missing: true - script: + tag: script_258616c7 description: Drops null/empty values recursively. lang: painless source: | @@ -516,4 +566,4 @@ on_failure: allow_duplicates: false - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' \ No newline at end of file + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/jamf_protect/data_stream/alerts/elasticsearch/ingest_pipeline/gpunifiedlogevent.yml b/packages/jamf_protect/data_stream/alerts/elasticsearch/ingest_pipeline/gpunifiedlogevent.yml index 7972a47e4f7..9ad0a1f23b3 100644 --- a/packages/jamf_protect/data_stream/alerts/elasticsearch/ingest_pipeline/gpunifiedlogevent.yml +++ b/packages/jamf_protect/data_stream/alerts/elasticsearch/ingest_pipeline/gpunifiedlogevent.yml @@ -2,28 +2,35 @@ description: Pipeline for processing GPUnifiedLogEvent events. processors: -###################### -## ECS Log Mapping ## -###################### + ###################### + ## ECS Log Mapping ## + ###################### - rename: + tag: rename_jamf_protect_alerts_input_match_event_process_to_process_name_5aa43c89 field: jamf_protect.alerts.input.match.event.process target_field: process.name if: ctx.jamf_protect?.alerts?.input?.eventType == "GPUnifiedLogEvent" && ctx.jamf_protect?.alerts?.input?.match?.event?.process != null - rename: + tag: rename_jamf_protect_alerts_input_match_event_processIdentifier_to_process_pid_e3218ce5 field: jamf_protect.alerts.input.match.event.processIdentifier target_field: process.pid if: ctx.jamf_protect?.alerts?.input?.eventType == "GPUnifiedLogEvent" && ctx.jamf_protect?.alerts?.input?.match?.event?.processIdentifier != null - date: + tag: date_jamf_protect_alerts_input_match_event_timestamp_to_process_start_c6f6e97a field: jamf_protect.alerts.input.match.event.timestamp target_field: process.start - formats: + formats: - UNIX if: ctx.jamf_protect?.alerts?.input?.eventType == "GPUnifiedLogEvent" && ctx.jamf_protect?.alerts?.input?.match?.event?.timestamp != null on_failure: +- append: + field: tags + value: preserve_original_event + allow_duplicates: false - set: field: event.kind value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' \ No newline at end of file + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/jamf_protect/data_stream/alerts/elasticsearch/ingest_pipeline/gpusbevent.yml b/packages/jamf_protect/data_stream/alerts/elasticsearch/ingest_pipeline/gpusbevent.yml index c8cd62db537..9a62c8a5907 100644 --- a/packages/jamf_protect/data_stream/alerts/elasticsearch/ingest_pipeline/gpusbevent.yml +++ b/packages/jamf_protect/data_stream/alerts/elasticsearch/ingest_pipeline/gpusbevent.yml @@ -2,58 +2,73 @@ description: Pipeline for processing GPUSBEvent events. processors: -###################### -## ECS Volume Mapping ## -###################### + ###################### + ## ECS Volume Mapping ## + ###################### - rename: + tag: rename_jamf_protect_alerts_input_match_event_device_bsdName_to_volume_nt_name_5755054e field: jamf_protect.alerts.input.match.event.device.bsdName target_field: volume.nt_name if: ctx.jamf_protect?.alerts?.input?.eventType == "GPUSBEvent" && ctx.jamf_protect?.alerts?.input?.match?.event?.device?.bsdName != null - rename: + tag: rename_jamf_protect_alerts_input_match_event_device_content_to_volume_file_system_type_f4851333 field: jamf_protect.alerts.input.match.event.device.content target_field: volume.file_system_type if: ctx.jamf_protect?.alerts?.input?.eventType == "GPUSBEvent" && ctx.jamf_protect?.alerts?.input?.match?.event?.device?.content != null - rename: + tag: rename_jamf_protect_alerts_input_match_event_device_busName_to_volume_bus_type_54031277 field: jamf_protect.alerts.input.match.event.device.busName target_field: volume.bus_type if: ctx.jamf_protect?.alerts?.input?.eventType == "GPUSBEvent" && ctx.jamf_protect?.alerts?.input?.match?.event?.device?.busName != null - rename: + tag: rename_jamf_protect_alerts_input_match_event_device_productName_to_volume_product_name_d6550173 field: jamf_protect.alerts.input.match.event.device.productName target_field: volume.product_name if: ctx.jamf_protect?.alerts?.input?.eventType == "GPUSBEvent" && ctx.jamf_protect?.alerts?.input?.match?.event?.device?.productName != null - rename: + tag: rename_jamf_protect_alerts_input_match_event_device_productId_to_volume_product_id_63d39b5d field: jamf_protect.alerts.input.match.event.device.productId target_field: volume.product_id if: ctx.jamf_protect?.alerts?.input?.eventType == "GPUSBEvent" && ctx.jamf_protect?.alerts?.input?.match?.event?.device?.productId != null - rename: + tag: rename_jamf_protect_alerts_input_match_event_device_isRemovable_to_volume_removable_3308945d field: jamf_protect.alerts.input.match.event.device.isRemovable target_field: volume.removable if: ctx.jamf_protect?.alerts?.input?.eventType == "GPUSBEvent" && ctx.jamf_protect?.alerts?.input?.match?.event?.device?.isRemovable != null - rename: + tag: rename_jamf_protect_alerts_input_match_event_device_serialNumber_to_volume_serial_number_fe028690 field: jamf_protect.alerts.input.match.event.device.serialNumber target_field: volume.serial_number if: ctx.jamf_protect?.alerts?.input?.eventType == "GPUSBEvent" && ctx.jamf_protect?.alerts?.input?.match?.event?.device?.serialNumber != null - rename: + tag: rename_jamf_protect_alerts_input_match_event_device_size_to_volume_size_d705e483 field: jamf_protect.alerts.input.match.event.device.size target_field: volume.size if: ctx.jamf_protect?.alerts?.input?.eventType == "GPUSBEvent" && ctx.jamf_protect?.alerts?.input?.match?.event?.device?.size != null - rename: + tag: rename_jamf_protect_alerts_input_match_event_device_vendorId_to_volume_vendor_id_575bc622 field: jamf_protect.alerts.input.match.event.device.vendorId target_field: volume.vendor_id if: ctx.jamf_protect?.alerts?.input?.eventType == "GPUSBEvent" && ctx.jamf_protect?.alerts?.input?.match?.event?.device?.vendorId != null - rename: + tag: rename_jamf_protect_alerts_input_match_event_device_vendorName_to_volume_vendor_name_41849a08 field: jamf_protect.alerts.input.match.event.device.vendorName target_field: volume.vendor_name if: ctx.jamf_protect?.alerts?.input?.eventType == "GPUSBEvent" && ctx.jamf_protect?.alerts?.input?.match?.event?.device?.vendorName != null - rename: + tag: rename_jamf_protect_alerts_input_match_event_device_isWritable_to_volume_writable_e986f1de field: jamf_protect.alerts.input.match.event.device.isWritable target_field: volume.writable if: ctx.jamf_protect?.alerts?.input?.eventType == "GPUSBEvent" && ctx.jamf_protect?.alerts?.input?.match?.event?.device?.isWritable != null on_failure: +- append: + field: tags + value: preserve_original_event + allow_duplicates: false - set: field: event.kind value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' \ No newline at end of file + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/default.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/default.yml index d55bcc9215a..a88e934852f 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/default.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/default.yml @@ -2,69 +2,85 @@ description: Pipeline for Jamf Protect Telemetry logs. processors: - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - json: + tag: json_message_to_jamf_protect_telemetry_9dae2753 field: message target_field: jamf_protect.telemetry if: ctx.message != null - rename: + tag: rename_json_to_jamf_protect_telemetry_1146a989 field: json target_field: jamf_protect.telemetry if: ctx.json != null -########################## -## ECS Observer Mapping ## -########################## + ########################## + ## ECS Observer Mapping ## + ########################## - set: + tag: set_observer_product_e40a396c field: observer.product value: Jamf Protect - set: + tag: set_observer_vendor_6a522e0a field: observer.vendor value: Jamf - set: + tag: set_observer_type_ae54f4bd field: observer.type value: Endpoint Security - rename: + tag: rename_jamf_protect_telemetry_host_protectVersion_to_observer_version_0472c50c field: jamf_protect.telemetry.host.protectVersion target_field: observer.version ignore_missing: true -####################### -## ECS Event Mapping ## -####################### + ####################### + ## ECS Event Mapping ## + ####################### - set: + tag: set_event_kind_de80643c field: event.kind value: event - append: + tag: append_event_type_8a66ccaa field: event.type value: info - append: + tag: append_event_category_8a525c81 field: event.category value: process - set: + tag: set_event_provider_7e881320 field: event.provider value: Jamf Protect - date: + tag: date_jamf_protect_telemetry_time_to_event_start_4eab876b field: jamf_protect.telemetry.time target_field: event.start - formats: + formats: - date_optional_time if: ctx.jamf_protect?.telemetry?.time != null - rename: + tag: rename_jamf_protect_telemetry_uuid_to_event_id_b6cdc923 field: jamf_protect.telemetry.uuid target_field: event.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_glob_seq_num_to_event_sequence_9b2ce931 field: jamf_protect.telemetry.glob_seq_num target_field: event.sequence ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_type_to_event_code_692889d5 field: jamf_protect.telemetry.event_type target_field: event.code type: string ignore_missing: true - script: + tag: script_86b62266 description: Populates event.action with the name of the event lang: painless source: > @@ -82,6 +98,7 @@ processors: } ignore_failure: true - script: + tag: script_1e0e4d20 lang: painless params: authTypeMap: @@ -94,255 +111,325 @@ processors: ctx.jamf_protect.telemetry = ctx.jamf_protect.telemetry != null ? ctx.jamf_protect.telemetry : new HashMap(); ctx.jamf_protect.telemetry.event_allowed_by_esclient = authTypeBoolean; } -####################### -## ECS Host Mapping ## -####################### + ####################### + ## ECS Host Mapping ## + ####################### - rename: + tag: rename_jamf_protect_telemetry_host_hostname_to_host_hostname_c3e1e88f field: jamf_protect.telemetry.host.hostname target_field: host.hostname ignore_missing: true - lowercase: + tag: lowercase_host_hostname_to_host_name_019f94e7 field: host.hostname target_field: host.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_host_provisioningUDID_to_host_id_0d0c735f field: jamf_protect.telemetry.host.provisioningUDID target_field: host.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_host_ips_to_host_ip_2ad85343 field: jamf_protect.telemetry.host.ips target_field: host.ip if: ctx.jamf_protect?.telemetry?.host?.ips != null && ctx.jamf_protect?.telemetry?.host?.ips != "" - rename: + tag: rename_jamf_protect_telemetry_host_os_to_host_os_full_54d0a856 field: jamf_protect.telemetry.host.os target_field: host.os.full ignore_missing: true - gsub: + tag: gsub_host_os_full_ba560804 field: host.os.full pattern: '^Version\s*' replacement: "" if: ctx.host?.os?.full != null - gsub: + tag: gsub_host_os_full_to_host_os_version_7562aa23 field: host.os.full pattern: ^([^\s]+).* replacement: "$1" target_field: host.os.version if: ctx.host?.os?.full != null - set: + tag: set_host_os_family_9d799ed3 field: host.os.family value: macos - set: + tag: set_host_os_type_074dd92b field: host.os.type value: macos - set: + tag: set_host_os_name_9a719836 field: host.os.name value: macOS -####################### -## ECS Device Mapping ## -####################### + ####################### + ## ECS Device Mapping ## + ####################### - rename: + tag: rename_jamf_protect_telemetry_host_serial_to_device_serial_number_b10070af field: jamf_protect.telemetry.host.serial target_field: device.serial_number ignore_missing: true - set: + tag: set_device_id_49c80c98 field: device.id value: '{{{ device.serial_number }}}' if: ctx.device?.serial_number != null ignore_failure: true - set: + tag: set_device_manufacturer_6c63de55 field: device.manufacturer value: Apple -######################## -# ECS Related Mapping ## -######################## + ######################## + # ECS Related Mapping ## + ######################## - append: + tag: append_related_hosts_122a4356 field: related.hosts value: '{{{ host.hostname }}}' if: ctx.host?.hostname != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_981c7fa7 field: related.user value: "{{{ user.name }}}" if: ctx.user?.name != null allow_duplicates: false - foreach: + tag: foreach_host_ip_232e838f field: host.ip if: ctx.host?.ip instanceof List processor: - append: + append: field: related.ip value: '{{{_ingest._value}}}' allow_duplicates: false -####################### -## ECS Pipelines ## -####################### + ####################### + ## ECS Pipelines ## + ####################### - pipeline: + tag: pipeline_253d06b8 name: '{{ IngestPipeline "pipeline_event_exec" }}' if: ctx.event.action == "exec" - pipeline: + tag: pipeline_1b7c268e name: '{{ IngestPipeline "pipeline_event_kextload" }}' if: ctx.event.action == "kextload" - pipeline: + tag: pipeline_f32dc118 name: '{{ IngestPipeline "pipeline_event_kextunload" }}' if: ctx.event.action == "kextunload" - pipeline: + tag: pipeline_58d559d0 name: '{{ IngestPipeline "pipeline_event_mount" }}' if: ctx.event.action == "mount" - pipeline: + tag: pipeline_9fbbbcb4 name: '{{ IngestPipeline "pipeline_event_unmount" }}' if: ctx.event.action == "unmount" - pipeline: + tag: pipeline_31719978 name: '{{ IngestPipeline "pipeline_event_chroot" }}' if: ctx.event.action == "chroot" - pipeline: + tag: pipeline_fb757fd8 name: '{{ IngestPipeline "pipeline_event_settime" }}' if: ctx.event.action == "settime" - pipeline: + tag: pipeline_1e960e92 name: '{{ IngestPipeline "pipeline_event_cs_invalidated" }}' if: ctx.event.action == "cs_invalidated" - pipeline: + tag: pipeline_aa4f9c08 name: '{{ IngestPipeline "pipeline_event_remount" }}' if: ctx.event.action == "remount" - pipeline: + tag: pipeline_71a1cd2c name: '{{ IngestPipeline "pipeline_event_xp_malware_detected" }}' if: ctx.event.action == "xp_malware_detected" - pipeline: + tag: pipeline_75dbb530 name: '{{ IngestPipeline "pipeline_event_xp_malware_remediated" }}' if: ctx.event.action == "xp_malware_remediated" - pipeline: + tag: pipeline_b94bf586 name: '{{ IngestPipeline "pipeline_event_lw_session_login" }}' if: ctx.event.action == "lw_session_login" - pipeline: + tag: pipeline_8bc6bc18 name: '{{ IngestPipeline "pipeline_event_lw_session_logout" }}' if: ctx.event.action == "lw_session_logout" - pipeline: + tag: pipeline_2393f0dc name: '{{ IngestPipeline "pipeline_event_lw_session_lock" }}' if: ctx.event.action == "lw_session_lock" - pipeline: + tag: pipeline_45b23da8 name: '{{ IngestPipeline "pipeline_event_lw_session_unlock" }}' if: ctx.event.action == "lw_session_unlock" - pipeline: + tag: pipeline_ab9c67ae name: '{{ IngestPipeline "pipeline_event_screensharing_attach" }}' if: ctx.event.action == "screensharing_attach" - pipeline: + tag: pipeline_fc30f536 name: '{{ IngestPipeline "pipeline_event_screensharing_detach" }}' if: ctx.event.action == "screensharing_detach" - pipeline: + tag: pipeline_789af360 name: '{{ IngestPipeline "pipeline_event_openssh_login" }}' if: ctx.event.action == "openssh_login" - pipeline: + tag: pipeline_3ea21b1c name: '{{ IngestPipeline "pipeline_event_openssh_logout" }}' if: ctx.event.action == "openssh_logout" - pipeline: + tag: pipeline_0258c114 name: '{{ IngestPipeline "pipeline_event_login_login" }}' if: ctx.event.action == "login_login" - pipeline: + tag: pipeline_72a3a90e name: '{{ IngestPipeline "pipeline_event_login_logout" }}' if: ctx.event.action == "login_logout" - pipeline: + tag: pipeline_d81048d8 name: '{{ IngestPipeline "pipeline_event_btm_launch_item_add" }}' if: ctx.event.action == "btm_launch_item_add" - pipeline: + tag: pipeline_d739e7da name: '{{ IngestPipeline "pipeline_event_btm_launch_item_remove" }}' if: ctx.event.action == "btm_launch_item_remove" - pipeline: + tag: pipeline_7ee10378 name: '{{ IngestPipeline "pipeline_event_profile_add" }}' if: ctx.event.action == "profile_add" - pipeline: + tag: pipeline_5f3c960a name: '{{ IngestPipeline "pipeline_event_profile_remove" }}' if: ctx.event.action == "profile_remove" - pipeline: + tag: pipeline_3620bcd6 name: '{{ IngestPipeline "pipeline_event_su" }}' if: ctx.event.action == "su" - pipeline: + tag: pipeline_f4da84f2 name: '{{ IngestPipeline "pipeline_event_authentication" }}' if: ctx.event.action == "authentication" - pipeline: + tag: pipeline_0d68c9ac name: '{{ IngestPipeline "pipeline_event_sudo" }}' if: ctx.event.action == "sudo" - pipeline: + tag: pipeline_ea4e9b28 name: '{{ IngestPipeline "pipeline_event_od_group_add" }}' if: ctx.event.action == "od_group_add" - pipeline: + tag: pipeline_0c635268 name: '{{ IngestPipeline "pipeline_event_od_group_remove" }}' if: ctx.event.action == "od_group_remove" - pipeline: + tag: pipeline_3fc1dd8a name: '{{ IngestPipeline "pipeline_event_od_group_set" }}' if: ctx.event.action == "od_group_set" - pipeline: + tag: pipeline_4af15172 name: '{{ IngestPipeline "pipeline_event_od_modify_password" }}' if: ctx.event.action == "od_modify_password" - pipeline: + tag: pipeline_a80e3e68 name: '{{ IngestPipeline "pipeline_event_od_disable_user" }}' if: ctx.event.action == "od_disable_user" - pipeline: + tag: pipeline_73f9bf38 name: '{{ IngestPipeline "pipeline_event_od_enable_user" }}' if: ctx.event.action == "od_enable_user" - pipeline: + tag: pipeline_990b48d6 name: '{{ IngestPipeline "pipeline_event_od_attribute_value_add" }}' if: ctx.event.action == "od_attribute_value_add" - pipeline: + tag: pipeline_f68e9568 name: '{{ IngestPipeline "pipeline_event_od_attribute_value_remove" }}' if: ctx.event.action == "od_attribute_value_remove" - pipeline: + tag: pipeline_204bdc28 name: '{{ IngestPipeline "pipeline_event_od_attribute_set" }}' if: ctx.event.action == "od_attribute_set" - pipeline: + tag: pipeline_2f7f782a name: '{{ IngestPipeline "pipeline_event_od_create_user" }}' if: ctx.event.action == "od_create_user" - pipeline: + tag: pipeline_e17140c4 name: '{{ IngestPipeline "pipeline_event_od_create_group" }}' if: ctx.event.action == "od_create_group" - pipeline: + tag: pipeline_3d32af60 name: '{{ IngestPipeline "pipeline_event_od_delete_user" }}' if: ctx.event.action == "od_delete_user" - pipeline: + tag: pipeline_442914e8 name: '{{ IngestPipeline "pipeline_event_od_delete_group" }}' if: ctx.event.action == "od_delete_group" - pipeline: + tag: pipeline_37190c98 name: '{{ IngestPipeline "pipeline_event_log_collection" }}' if: ctx.event.action == "log_collection" - pipeline: + tag: pipeline_ed2f8654 name: '{{ IngestPipeline "pipeline_event_file_collection" }}' if: ctx.event.action == "file_collection" - pipeline: + tag: pipeline_c97d1942 name: '{{ IngestPipeline "pipeline_event_system_performance" }}' if: ctx.event.action == "system_performance" - pipeline: + tag: pipeline_b51c0114 name: '{{ IngestPipeline "pipeline_event_bios_uefi" }}' if: ctx.event.action == "bios_uefi" - pipeline: + tag: pipeline_ea04e202 name: '{{ IngestPipeline "pipeline_event_gatekeeper_user_override" }}' if: ctx.event.action == "gatekeeper_user_override" - pipeline: + tag: pipeline_fcbe207c name: '{{ IngestPipeline "pipeline_event_network_connect" }}' if: ctx.event.action == "network_connect" - pipeline: + tag: pipeline_d0d08af0 name: '{{ IngestPipeline "pipeline_event_tcc_modify" }}' if: ctx.event.action == "tcc_modify" - pipeline: + tag: pipeline_6e20b148 name: '{{ IngestPipeline "pipeline_event_pty_grant" }}' if: ctx.event.action == "pty_grant" - pipeline: + tag: pipeline_4a7d80a8 name: '{{ IngestPipeline "pipeline_event_pty_close" }}' if: ctx.event.action == "pty_close" - convert: + tag: convert_process_code_signature_flags_11d61406 field: process.code_signature.flags type: string ignore_missing: true - convert: + tag: convert_process_parent_code_signature_flags_c94f4f42 field: process.parent.code_signature.flags type: string ignore_missing: true - convert: + tag: convert_process_responsible_code_signature_flags_8dc74a2c field: process.responsible.code_signature.flags type: string ignore_missing: true -############# -## Cleanup ## -############# + ############# + ## Cleanup ## + ############# - remove: + tag: remove_3870c653 field: # - jamf_protect.telemetry - jamf_protect.telemetry.event @@ -367,6 +454,7 @@ processors: - message ignore_missing: true - script: + tag: script_e2ccbdd7 description: Drops null/empty values recursively. lang: painless source: | diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_authentication.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_authentication.yml index 2f3eb26faf8..ed5f27ea2d3 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_authentication.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_authentication.yml @@ -2,13 +2,15 @@ description: Pipeline for parsing specific fields related to authentication events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - append: + tag: append_event_category_109b4d72 field: event.category value: authentication - script: + tag: script_b4792e37 lang: painless params: itemTypeMap: @@ -25,10 +27,12 @@ processors: } ignore_failure: true - script: + tag: script_a371b2cb lang: painless source: > ctx.event.reason = 'A user authentication happened using ' + ctx.jamf_protect.telemetry.authentication_method; - script: + tag: script_6f1ad918 lang: painless source: | ctx.event = ctx.event != null ? ctx.event : new HashMap(); @@ -44,6 +48,7 @@ processors: } ignore_failure: true - script: + tag: script_c490f4c4 lang: painless params: itemTypeMap: @@ -58,28 +63,34 @@ processors: } ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_record_name_to_user_name_72a42c27 field: jamf_protect.telemetry.event.authentication.data.od.record_name target_field: user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_db_path_to_file_path_39c735d3 field: jamf_protect.telemetry.event.authentication.data.od.db_path target_field: file.path ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_od_record_type_to_jamf_protect_telemetry_record_type_8d15ff85 field: jamf_protect.telemetry.event.authentication.data.od.record_type target_field: jamf_protect.telemetry.record_type type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_username_to_user_name_68e770d5 field: jamf_protect.telemetry.event.authentication.data.auto_unlock.username target_field: user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_token_kerberos_principal_to_jamf_protect_telemetry_authentication_token_kerberos_principal_b3479a70 field: jamf_protect.telemetry.event.authentication.data.token.kerberos_principal target_field: jamf_protect.telemetry.authentication_token_kerberos_principal type: string ignore_missing: true - script: + tag: script_9c04d7d1 lang: painless params: itemTypeMap: @@ -94,91 +105,106 @@ processors: } ignore_failure: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_instigator_start_time_to_process_start_4f435545 field: jamf_protect.telemetry.event.authentication.data.od.instigator.start_time target_field: process.start ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_instigator_start_time_to_process_start_c80d0b06 field: jamf_protect.telemetry.event.authentication.data.token.instigator.start_time target_field: process.start ignore_missing: true if: ctx.process?.start == null - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_instigator_start_time_to_process_start_644829ed field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.start_time target_field: process.start ignore_missing: true if: ctx.process?.start == null - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_start_time_to_process_start_d62adc6b field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.start_time target_field: process.start ignore_missing: true if: ctx.process?.start == null - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_od_instigator_audit_token_egid_to_jamf_protect_telemetry_event_authentication_instigator_audit_token_egid_ec167595 field: jamf_protect.telemetry.event.authentication.data.od.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.authentication.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_token_instigator_audit_token_egid_to_jamf_protect_telemetry_event_authentication_instigator_audit_token_egid_abd25911 field: jamf_protect.telemetry.event.authentication.data.token.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.authentication.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_touchid_instigator_audit_token_egid_to_jamf_protect_telemetry_event_authentication_instigator_audit_token_egid_f2238d36 field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.authentication.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_auto_unock_instigator_audit_token_egid_to_jamf_protect_telemetry_event_authentication_instigator_audit_token_egid_58ffeb38 field: jamf_protect.telemetry.event.authentication.data.auto_unock.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.authentication.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_od_instigator_audit_token_euid_to_jamf_protect_telemetry_event_authentication_instigator_audit_token_euid_a34f0851 field: jamf_protect.telemetry.event.authentication.data.od.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.authentication.instigator.audit_token.euid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_token_instigator_audit_token_euid_to_jamf_protect_telemetry_event_authentication_instigator_audit_token_euid_e360d92d field: jamf_protect.telemetry.event.authentication.data.token.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.authentication.instigator.audit_token.euid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_touchid_instigator_audit_token_euid_to_jamf_protect_telemetry_event_authentication_instigator_audit_token_euid_441e89e6 field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.authentication.instigator.audit_token.euid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_audit_token_euid_to_jamf_protect_telemetry_event_authentication_instigator_audit_token_euid_e61aa64c field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.authentication.instigator.audit_token.euid type: string ignore_missing: true - - append: + tag: append_user_effective_id_1f58ed7d field: user.effective.id value: '{{{jamf_protect.telemetry.event.authentication.data.od.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.authentication?.data?.od?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - append: + tag: append_user_effective_id_b4e558f3 field: user.effective.id value: '{{{jamf_protect.telemetry.event.authentication.data.token.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.authentication?.data?.token?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - append: + tag: append_user_effective_id_54fa05b9 field: user.effective.id value: '{{{jamf_protect.telemetry.event.authentication.data.touchid.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.authentication?.data?.touchid?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - append: + tag: append_user_effective_id_bb9e4d29 field: user.effective.id value: '{{{jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.authentication?.data?.auto_unlock?.instigator?.audit_token?.euid != null @@ -186,245 +212,301 @@ processors: ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_instigator_audit_token_e_username_to_process_user_name_1dcb511c field: jamf_protect.telemetry.event.authentication.data.od.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_instigator_audit_token_e_username_to_process_user_name_1f96e81e field: jamf_protect.telemetry.event.authentication.data.token.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_instigator_audit_token_e_username_to_process_user_name_ab93db6d field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_audit_token_e_username_to_process_user_name_8ae1c733 field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_instigator_is_platform_binary_to_process_platform_binary_4c7ef8b8 field: jamf_protect.telemetry.event.authentication.data.od.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_instigator_is_platform_binary_to_process_platform_binary_46dfb066 field: jamf_protect.telemetry.event.authentication.data.token.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_instigator_is_platform_binary_to_process_platform_binary_4ac82e31 field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_is_platform_binary_to_process_platform_binary_23cbfe53 field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_instigator_is_es_client_to_process_endpoint_security_client_5051081a field: jamf_protect.telemetry.event.authentication.data.od.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_instigator_is_es_client_to_process_endpoint_security_client_ff32978c field: jamf_protect.telemetry.event.authentication.data.token.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_instigator_is_es_client_to_process_endpoint_security_client_63bdd291 field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_is_es_client_to_process_endpoint_security_client_2141c23f field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_instigator_cdhash_to_process_hash_cdhash_1c0b102d field: jamf_protect.telemetry.event.authentication.data.od.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_instigator_cdhash_to_process_hash_cdhash_e4fcc1e3 field: jamf_protect.telemetry.event.authentication.data.token.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_instigator_cdhash_to_process_hash_cdhash_c55eee94 field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_cdhash_to_process_hash_cdhash_f372cc5e field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_instigator_executable_sha1_to_process_hash_sha1_a5535a45 field: jamf_protect.telemetry.event.authentication.data.od.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_instigator_executable_sha1_to_process_hash_sha1_e0e1a483 field: jamf_protect.telemetry.event.authentication.data.token.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_instigator_executable_sha1_to_process_hash_sha1_25bab34a field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_executable_sha1_to_process_hash_sha1_be759fb4 field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_instigator_executable_sha256_to_process_hash_sha256_b8c53dbd field: jamf_protect.telemetry.event.authentication.data.od.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_instigator_executable_sha256_to_process_hash_sha256_35e2d11b field: jamf_protect.telemetry.event.authentication.data.token.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_instigator_executable_sha256_to_process_hash_sha256_7d03e722 field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_executable_sha256_to_process_hash_sha256_358957dc field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_instigator_tty_path_to_jamf_protect_telemetry_tty_0da15276 field: jamf_protect.telemetry.event.authentication.data.od.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_instigator_tty_path_to_jamf_protect_telemetry_tty_c86f2638 field: jamf_protect.telemetry.event.authentication.data.token.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_instigator_tty_path_to_jamf_protect_telemetry_tty_35c4f2ed field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_tty_path_to_jamf_protect_telemetry_tty_d40a2a83 field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_150b9a27 field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.authentication?.data?.od?.instigator?.tty != null - set: + tag: set_process_interactive_87c7729a field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.authentication?.data?.od?.instigator?.tty == null - set: + tag: set_process_interactive_9d506a47 field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.authentication?.data?.token?.instigator?.tty != null - set: + tag: set_process_interactive_032237ba field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.authentication?.data?.token?.instigator?.tty == null - set: + tag: set_process_interactive_73fb2b4a field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.authentication?.data?.touchid?.instigator?.tty != null - set: + tag: set_process_interactive_6521198d field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.authentication?.data?.touchid?.instigator?.tty == null - set: + tag: set_process_interactive_219355b4 field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.authentication?.data?.auto_unlock?.instigator?.tty != null - set: + tag: set_process_interactive_103b7adf field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.authentication?.data?.auto_unlock?.instigator?.tty == null - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_od_instigator_audit_token_pid_to_process_pid_37bb172a field: jamf_protect.telemetry.event.authentication.data.od.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_e66043d3 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_token_instigator_audit_token_pid_to_process_pid_e8f8ddde field: jamf_protect.telemetry.event.authentication.data.token.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_c1ced139 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_touchid_instigator_audit_token_pid_to_process_pid_7d573e05 field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_8e20fd3a field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_audit_token_pid_to_process_pid_b812e697 field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_0147f398 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_instigator_audit_token_uuid_to_process_entity_id_4b7d3b0c field: jamf_protect.telemetry.event.authentication.data.od.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_instigator_audit_token_uuid_to_process_entity_id_80b0596e field: jamf_protect.telemetry.event.authentication.data.token.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_instigator_audit_token_uuid_to_process_entity_id_3abee6e9 field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_audit_token_uuid_to_process_entity_id_f70f800f field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_instigator_executable_path_to_process_executable_1a603336 field: jamf_protect.telemetry.event.authentication.data.od.instigator.executable.path target_field: process.executable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_instigator_executable_path_to_process_executable_7e2e8c58 field: jamf_protect.telemetry.event.authentication.data.token.instigator.executable.path target_field: process.executable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_instigator_executable_path_to_process_executable_d5786cd7 field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.executable.path target_field: process.executable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_executable_path_to_process_executable_9a990ccd field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -437,363 +519,441 @@ processors: } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_instigator_signing_id_to_process_code_signature_signing_id_f0373949 field: jamf_protect.telemetry.event.authentication.data.od.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_instigator_signing_id_to_process_code_signature_signing_id_a64b2f07 field: jamf_protect.telemetry.event.authentication.data.token.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_instigator_signing_id_to_process_code_signature_signing_id_6716a31c field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_signing_id_to_process_code_signature_signing_id_4fe3670e field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_instigator_team_id_to_process_code_signature_team_id_46ee1d33 field: jamf_protect.telemetry.event.authentication.data.od.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_instigator_team_id_to_process_code_signature_team_id_c2a5db71 field: jamf_protect.telemetry.event.authentication.data.token.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_instigator_team_id_to_process_code_signature_team_id_40bd3a82 field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_team_id_to_process_code_signature_team_id_de550c4c field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_instigator_codesigning_flags_to_process_code_signature_flags_9dd637b6 field: jamf_protect.telemetry.event.authentication.data.od.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_instigator_codesigning_flags_to_process_code_signature_flags_08ae7614 field: jamf_protect.telemetry.event.authentication.data.token.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_instigator_codesigning_flags_to_process_code_signature_flags_1a0b2807 field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_codesigning_flags_to_process_code_signature_flags_6d7a4b39 field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_instigator_parent_audit_token_uuid_to_process_parent_entity_id_a21f0429 field: jamf_protect.telemetry.event.authentication.data.od.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_instigator_parent_audit_token_uuid_to_process_parent_entity_id_a41175c3 field: jamf_protect.telemetry.event.authentication.data.token.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_instigator_parent_audit_token_uuid_to_process_parent_entity_id_96f94a8c field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_parent_audit_token_uuid_to_process_parent_entity_id_9aeec632 field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_instigator_parent_audit_token_pid_to_process_parent_pid_416e8a29 field: jamf_protect.telemetry.event.authentication.data.od.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_instigator_parent_audit_token_pid_to_process_parent_pid_90b2334f field: jamf_protect.telemetry.event.authentication.data.token.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_instigator_parent_audit_token_pid_to_process_parent_pid_6beab29a field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_parent_audit_token_pid_to_process_parent_pid_bb60836c field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_od_instigator_parent_audit_token_euid_to_process_parent_user_id_ca757746 field: jamf_protect.telemetry.event.authentication.data.od.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_token_instigator_parent_audit_token_euid_to_process_parent_user_id_1c4581e2 field: jamf_protect.telemetry.event.authentication.data.token.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_touchid_instigator_parent_audit_token_euid_to_process_parent_user_id_0663ea19 field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_parent_audit_token_euid_to_process_parent_user_id_2c7a1043 field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_parent_audit_token_e_username_to_process_parent_user_name_a71076cf field: jamf_protect.telemetry.event.authentication.data.od.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_parent_audit_token_e_username_to_process_parent_user_name_ebaaae1d field: jamf_protect.telemetry.event.authentication.data.token.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_parent_audit_token_e_username_to_process_parent_user_name_91f40a30 field: jamf_protect.telemetry.event.authentication.data.touchid.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_parent_audit_token_e_username_to_process_parent_user_name_80604862 field: jamf_protect.telemetry.event.authentication.data.auto_unlock.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_od_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_3f9328fa field: jamf_protect.telemetry.event.authentication.data.od.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_token_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_8f10f2de field: jamf_protect.telemetry.event.authentication.data.token.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_touchid_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_b419e0fb field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_39a344a5 field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_od_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_c2222416 field: jamf_protect.telemetry.event.authentication.data.od.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_token_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_0d8ad89a field: jamf_protect.telemetry.event.authentication.data.token.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_touchid_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_1955c86d field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_27ec8ad3 field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_9f4b2609 field: jamf_protect.telemetry.event.authentication.data.od.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_bd4c6b6b field: jamf_protect.telemetry.event.authentication.data.token.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_5b702794 field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_10c643e2 field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_parent_audit_token_exec_path_to_process_parent_executable_3da09a56 field: jamf_protect.telemetry.event.authentication.data.od.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_parent_audit_token_exec_path_to_process_parent_executable_295b2074 field: jamf_protect.telemetry.event.authentication.data.token.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_parent_audit_token_exec_path_to_process_parent_executable_00d6d5d1 field: jamf_protect.telemetry.event.authentication.data.touchid.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_parent_audit_token_exec_path_to_process_parent_executable_5d1f888b field: jamf_protect.telemetry.event.authentication.data.auto_unlock.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_3f0f9c63 field: jamf_protect.telemetry.event.authentication.data.od.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_b6641375 field: jamf_protect.telemetry.event.authentication.data.token.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_df16b942 field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_ff437fc8 field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_instigator_responsible_audit_token_pid_to_process_responsible_pid_d1241227 field: jamf_protect.telemetry.event.authentication.data.od.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_instigator_responsible_audit_token_pid_to_process_responsible_pid_44072dcd field: jamf_protect.telemetry.event.authentication.data.token.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_instigator_responsible_audit_token_pid_to_process_responsible_pid_8d36a10c field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_responsible_audit_token_pid_to_process_responsible_pid_d0afdbd6 field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_od_instigator_responsible_audit_token_euid_to_process_responsible_user_id_95860784 field: jamf_protect.telemetry.event.authentication.data.od.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_token_instigator_responsible_audit_token_euid_to_process_responsible_user_id_6b3ad4b0 field: jamf_protect.telemetry.event.authentication.data.token.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_touchid_instigator_responsible_audit_token_euid_to_process_responsible_user_id_99aefb7b field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_responsible_audit_token_euid_to_process_responsible_user_id_46d94ead field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_responsible_audit_token_e_username_to_process_responsible_user_name_f037f9c7 field: jamf_protect.telemetry.event.authentication.data.od.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_responsible_audit_token_e_username_to_process_responsible_user_name_4fa421b5 field: jamf_protect.telemetry.event.authentication.data.token.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_responsible_audit_token_e_username_to_process_responsible_user_name_bf51aa80 field: jamf_protect.telemetry.event.authentication.data.touchid.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_responsible_audit_token_e_username_to_process_responsible_user_name_a96e572a field: jamf_protect.telemetry.event.authentication.data.auto_unlock.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_od_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_16bb0832 field: jamf_protect.telemetry.event.authentication.data.od.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_token_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_c6495e26 field: jamf_protect.telemetry.event.authentication.data.token.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_touchid_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_cb301f17 field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_2c3d4ec1 field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_od_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_d786939a field: jamf_protect.telemetry.event.authentication.data.od.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_token_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_d8e4453e field: jamf_protect.telemetry.event.authentication.data.token.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_touchid_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_56a5f021 field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_1ebf0ad7 field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_5f44bfd7 field: jamf_protect.telemetry.event.authentication.data.od.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_7a5bf641 field: jamf_protect.telemetry.event.authentication.data.token.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_01a5c21e field: jamf_protect.telemetry.event.authentication.data.touchid.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_384b6154 field: jamf_protect.telemetry.event.authentication.data.auto_unlock.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_od_responsible_audit_token_exec_path_to_process_responsible_executable_cfcb8398 field: jamf_protect.telemetry.event.authentication.data.od.responsible_audit_token.exec_path target_field: process.responsible.executable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_token_responsible_audit_token_exec_path_to_process_responsible_executable_a8846e2e field: jamf_protect.telemetry.event.authentication.data.token.responsible_audit_token.exec_path target_field: process.responsible.executable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_touchid_responsible_audit_token_exec_path_to_process_responsible_executable_9b3639c7 field: jamf_protect.telemetry.event.authentication.data.touchid.responsible_audit_token.exec_path target_field: process.responsible.executable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_authentication_data_auto_unlock_responsible_audit_token_exec_path_to_process_responsible_executable_0f7e3919 field: jamf_protect.telemetry.event.authentication.data.auto_unlock.responsible_audit_token.exec_path target_field: process.responsible.executable ignore_missing: true - pipeline: + tag: pipeline_d6c46bbc name: '{{ IngestPipeline "pipeline_object_process" }}' if: ctx.jamf_protect?.telemetry?.event?.authentication?.data?.od == null && ctx.jamf_protect?.telemetry?.event?.authentication?.data?.token == null && ctx.jamf_protect?.telemetry?.event?.authentication?.data?.touchid == null && ctx.jamf_protect?.telemetry?.event?.authentication?.data?.auto_unlock?.instigator == null diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_bios_uefi.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_bios_uefi.yml index ee8c1828905..a7f854e0676 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_bios_uefi.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_bios_uefi.yml @@ -2,21 +2,25 @@ description: Pipeline for parsing specific fields related to bios events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_ca173c9e field: event.reason value: Collection of bios data - rename: + tag: rename_jamf_protect_telemetry_event_bios_uefi_architecture_to_host_architecture_579e6ae2 field: jamf_protect.telemetry.event.bios_uefi.architecture target_field: host.architecture ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_bios_uefi_bios_firmware-version_to_jamf_protect_telemetry_bios_firmware_version_e4d7afde field: jamf_protect.telemetry.event.bios_uefi.bios.firmware-version target_field: jamf_protect.telemetry.bios_firmware_version ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_bios_uefi_bios_system-firmware-version_to_jamf_protect_telemetry_bios_system_firmware_version_d6e6610c field: jamf_protect.telemetry.event.bios_uefi.bios.system-firmware-version target_field: jamf_protect.telemetry.bios_system_firmware_version ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_btm_launch_item_add.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_btm_launch_item_add.yml index 697faa2d508..231d2ed60d2 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_btm_launch_item_add.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_btm_launch_item_add.yml @@ -2,51 +2,61 @@ description: Pipeline for parsing specific fields related to btm add events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_df136db5 field: event.reason value: Apple’s Background Task Manager notified that an item has been added - append: + tag: append_event_type_0602e5be field: event.type value: change - append: + tag: append_event_category_1c667e2c field: event.category - value: configuration + value: configuration - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_item_app_url_to_jamf_protect_telemetry_btm_item_app_url_c20c5fd4 field: jamf_protect.telemetry.event.btm_launch_item_add.item.app_url target_field: jamf_protect.telemetry.btm_item_app_url type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_item_item_url_to_jamf_protect_telemetry_btm_item_url_9c2e6c36 field: jamf_protect.telemetry.event.btm_launch_item_add.item.item_url target_field: jamf_protect.telemetry.btm_item_url type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_item_uid_to_jamf_protect_telemetry_btm_item_user_uid_3cf0ef70 field: jamf_protect.telemetry.event.btm_launch_item_add.item.uid target_field: jamf_protect.telemetry.btm_item_user_uid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_item_legacy_to_jamf_protect_telemetry_btm_item_is_legacy_048daf43 field: jamf_protect.telemetry.event.btm_launch_item_add.item.legacy target_field: jamf_protect.telemetry.btm_item_is_legacy type: boolean ignore_missing: true ignore_failure: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_item_managed_to_jamf_protect_telemetry_btm_item_is_managed_87d9afb5 field: jamf_protect.telemetry.event.btm_launch_item_add.item.managed target_field: jamf_protect.telemetry.btm_item_is_managed type: boolean ignore_missing: true ignore_failure: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_executable_path_to_jamf_protect_telemetry_btm_executable_path_a5f817cf field: jamf_protect.telemetry.event.btm_launch_item_add.executable_path target_field: jamf_protect.telemetry.btm_executable_path type: string ignore_missing: true - script: + tag: script_be56750e lang: painless params: itemTypeMap: @@ -63,173 +73,211 @@ processors: ctx.jamf_protect.telemetry.btm_item_type = itemTypeString; } -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_start_time_to_process_start_545e3648 field: jamf_protect.telemetry.event.btm_launch_item_add.app.start_time target_field: process.start ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_start_time_to_process_start_54f70a42 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.start_time target_field: process.start ignore_missing: true if: ctx.process?.start == null - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_app_audit_token_egid_to_jamf_protect_telemetry_event_btm_launch_item_add_app_audit_token_egid_da15094e field: jamf_protect.telemetry.event.btm_launch_item_add.app.audit_token.egid target_field: jamf_protect.telemetry.event.btm_launch_item_add.app.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_instigator_audit_token_egid_to_jamf_protect_telemetry_event_btm_launch_item_add_instigator_audit_token_egid_e57119b6 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.audit_token.egid type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_audit_token_e_username_to_process_user_name_4a3ad93a field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_audit_token_e_username_to_process_user_name_b820a149 field: jamf_protect.telemetry.event.btm_launch_item_add.app.audit_token.e_username target_field: process.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_app_audit_token_euid_to_jamf_protect_telemetry_event_btm_launch_item_add_app_audit_token_euid_e3e13c56 field: jamf_protect.telemetry.event.btm_launch_item_add.app.audit_token.euid target_field: jamf_protect.telemetry.event.btm_launch_item_add.app.audit_token.euid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_instigator_audit_token_euid_to_jamf_protect_telemetry_event_btm_launch_item_add_instigator_audit_token_euid_afa05d62 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.audit_token.euid type: string ignore_missing: true - append: + tag: append_user_effective_id_2766a729 field: user.effective.id value: '{{{jamf_protect.telemetry.event.btm_launch_item_add.app.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_add?.app?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - append: + tag: append_user_effective_id_48a702bb field: user.effective.id value: '{{{jamf_protect.telemetry.event.btm_launch_item_add.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_add?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_is_platform_binary_to_process_platform_binary_b73e6b45 field: jamf_protect.telemetry.event.btm_launch_item_add.app.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_is_platform_binary_to_process_platform_binary_19f1d74a field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_is_es_client_to_process_endpoint_security_client_ff226755 field: jamf_protect.telemetry.event.btm_launch_item_add.app.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_is_es_client_to_process_endpoint_security_client_1c451ab0 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_cdhash_to_process_hash_cdhash_d6988c48 field: jamf_protect.telemetry.event.btm_launch_item_add.app.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_cdhash_to_process_hash_cdhash_e1d99cf7 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_executable_sha1_to_process_hash_sha1_a761cbde field: jamf_protect.telemetry.event.btm_launch_item_add.app.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_executable_sha1_to_process_hash_sha1_3d919ab7 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_executable_sha256_to_process_hash_sha256_ef8be396 field: jamf_protect.telemetry.event.btm_launch_item_add.app.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_executable_sha256_to_process_hash_sha256_2037a84f field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_tty_path_to_jamf_protect_telemetry_tty_2b4a4159 field: jamf_protect.telemetry.event.btm_launch_item_add.app.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_tty_path_to_jamf_protect_telemetry_tty_e162a504 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_b6ee088c field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_add?.app?.tty != null - set: + tag: set_process_interactive_6f6f127f field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_add?.instigator?.tty != null - set: + tag: set_process_interactive_57ca4ec7 field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_add?.app?.tty == null - set: + tag: set_process_interactive_a2e37b62 field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_add?.instigator?.tty == null - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_app_audit_token_pid_to_process_pid_0928207b field: jamf_protect.telemetry.event.btm_launch_item_add.app.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_210d52a6 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_instigator_audit_token_pid_to_process_pid_f3cdb8b6 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_3e525db1 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_audit_token_uuid_to_process_entity_id_ca5d9be5 field: jamf_protect.telemetry.event.btm_launch_item_add.app.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_audit_token_uuid_to_process_entity_id_b4b227ea field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_executable_path_to_process_executable_e57d03e3 field: jamf_protect.telemetry.event.btm_launch_item_add.app.executable.path target_field: process.executable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_executable_path_to_process_executable_b901d674 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -242,204 +290,244 @@ processors: } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_signing_id_to_process_code_signature_signing_id_3bf1ff78 field: jamf_protect.telemetry.event.btm_launch_item_add.app.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_signing_id_to_process_code_signature_signing_id_c015b7b3 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_team_id_to_process_code_signature_team_id_ca4fd38e field: jamf_protect.telemetry.event.btm_launch_item_add.app.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_team_id_to_process_code_signature_team_id_02ff044d field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_codesigning_flags_to_process_code_signature_flags_1f0c5fc3 field: jamf_protect.telemetry.event.btm_launch_item_add.app.codesigning_flags target_field: process.code_signature.flags ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_codesigning_flags_to_process_code_signature_flags_d5ff65e0 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_codesigning_flags_to_process_code_signature_flags_d50e190d field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.codesigning_flags target_field: process.code_signature.flags if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_add?.app == null ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_parent_audit_token_uuid_to_process_parent_entity_id_5c62d428 field: jamf_protect.telemetry.event.btm_launch_item_add.app.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_parent_audit_token_uuid_to_process_parent_entity_id_7a46ef0f field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_parent_audit_token_pid_to_process_parent_pid_223e0406 field: jamf_protect.telemetry.event.btm_launch_item_add.app.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_parent_audit_token_pid_to_process_parent_pid_54551f2b field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_app_parent_audit_token_euid_to_process_parent_user_id_ff09abaf field: jamf_protect.telemetry.event.btm_launch_item_add.app.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_instigator_parent_audit_token_euid_to_process_parent_user_id_14cbbaca field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_parent_audit_token_e_username_to_process_parent_user_name_cf6c908a field: jamf_protect.telemetry.event.btm_launch_item_add.app.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_parent_audit_token_e_username_to_process_parent_user_name_e8a08411 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_app_parent_audit_token_ruid_to_process_parent_real_user_id_99cc7909 field: jamf_protect.telemetry.event.btm_launch_item_add.app.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_927ceef6 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_app_parent_audit_token_rgid_to_process_parent_real_group_id_7462f627 field: jamf_protect.telemetry.event.btm_launch_item_add.app.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_e87f8ab2 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_409ffeb8 field: jamf_protect.telemetry.event.btm_launch_item_add.app.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_49e9edaf field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_parent_audit_token_exec_path_to_process_parent_executable_cf699503 field: jamf_protect.telemetry.event.btm_launch_item_add.app.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_parent_audit_token_exec_path_to_process_parent_executable_32cd6090 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## - + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_responsible_audit_token_uuid_to_process_responsible_entity_id_32972afe field: jamf_protect.telemetry.event.btm_launch_item_add.app.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_7173ce01 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_responsible_audit_token_pid_to_process_responsible_pid_9e6b18e8 field: jamf_protect.telemetry.event.btm_launch_item_add.app.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_responsible_audit_token_pid_to_process_responsible_pid_97e5c4f9 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_app_responsible_audit_token_euid_to_process_responsible_user_id_9d2a05c9 field: jamf_protect.telemetry.event.btm_launch_item_add.app.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_instigator_responsible_audit_token_euid_to_process_responsible_user_id_636b1eb8 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_responsible_audit_token_e_username_to_process_responsible_user_name_e21daf52 field: jamf_protect.telemetry.event.btm_launch_item_add.app.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_responsible_audit_token_e_username_to_process_responsible_user_name_cd966199 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_app_responsible_audit_token_ruid_to_process_responsible_real_user_id_1ec1bbd5 field: jamf_protect.telemetry.event.btm_launch_item_add.app.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_d4f11a9e field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_app_responsible_audit_token_rgid_to_process_responsible_real_group_id_54e0353b field: jamf_protect.telemetry.event.btm_launch_item_add.app.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_add_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_47f319d6 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_aac754f2 field: jamf_protect.telemetry.event.btm_launch_item_add.app.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_698bdc25 field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_app_responsible_audit_token_exec_path_to_process_responsible_executable_5c28cbb1 field: jamf_protect.telemetry.event.btm_launch_item_add.app.responsible_audit_token.exec_path target_field: process.responsible.executable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_add_instigator_responsible_audit_token_exec_path_to_process_responsible_executable_2a46dbfa field: jamf_protect.telemetry.event.btm_launch_item_add.instigator.responsible_audit_token.exec_path target_field: process.responsible.executable ignore_missing: true - pipeline: + tag: pipeline_d7666874 name: '{{ IngestPipeline "pipeline_object_process" }}' if: ctx.jamf_protect?.telemetry?.event?.btm_launch_item_add?.instigator == null && ctx.jamf_protect?.telemetry?.event?.btm_launch_item_add?.app == null diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_btm_launch_item_remove.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_btm_launch_item_remove.yml index 01c40d8c15a..df3bdf9ab20 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_btm_launch_item_remove.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_btm_launch_item_remove.yml @@ -2,46 +2,55 @@ description: Pipeline for parsing specific fields related to btm remove events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_0e67d4fc field: event.reason value: Apple’s Background Task Manager notified that an existing item has been removed - append: + tag: append_event_type_0602e5be field: event.type value: change - append: + tag: append_event_category_1c667e2c field: event.category - value: configuration + value: configuration - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_item_app_url_to_jamf_protect_telemetry_btm_item_app_url_403474bb field: jamf_protect.telemetry.event.btm_launch_item_remove.item.app_url target_field: jamf_protect.telemetry.btm_item_app_url type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_item_item_url_to_jamf_protect_telemetry_btm_item_url_41c777b7 field: jamf_protect.telemetry.event.btm_launch_item_remove.item.item_url target_field: jamf_protect.telemetry.btm_item_url type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_item_uid_to_jamf_protect_telemetry_btm_item_user_uid_09ee100d field: jamf_protect.telemetry.event.btm_launch_item_remove.item.uid target_field: jamf_protect.telemetry.btm_item_user_uid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_item_legacy_to_jamf_protect_telemetry_btm_item_is_legacy_c6f2d104 field: jamf_protect.telemetry.event.btm_launch_item_remove.item.legacy target_field: jamf_protect.telemetry.btm_item_is_legacy type: boolean ignore_missing: true ignore_failure: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_item_managed_to_jamf_protect_telemetry_btm_item_is_managed_bc482dce field: jamf_protect.telemetry.event.btm_launch_item_remove.item.managed target_field: jamf_protect.telemetry.btm_item_is_managed type: boolean ignore_missing: true ignore_failure: true - script: + tag: script_4c4c8b58 lang: painless params: itemTypeMap: @@ -58,164 +67,199 @@ processors: ctx.jamf_protect.telemetry.btm_item_type = itemTypeString; } -########################## -## ECS Process ## -########################## + + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_start_time_to_process_start_88623681 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.start_time target_field: process.start ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_start_time_to_process_start_71da5a46 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.start_time if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_app_audit_token_egid_to_jamf_protect_telemetry_event_btm_launch_item_remove_app_audit_token_egid_f0d4c5fe field: jamf_protect.telemetry.event.btm_launch_item_remove.app.audit_token.egid target_field: jamf_protect.telemetry.event.btm_launch_item_remove.app.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_audit_token_egid_to_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_audit_token_egid_fdfa48ec field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.audit_token.egid - if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null + if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_audit_token_e_username_to_process_user_name_ee36aa45 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_audit_token_e_username_to_process_user_name_50e82018 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.audit_token.e_username target_field: process.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_app_audit_token_euid_to_jamf_protect_telemetry_event_btm_launch_item_remove_app_audit_token_euid_03e32846 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.audit_token.euid target_field: jamf_protect.telemetry.event.btm_launch_item_remove.app.audit_token.euid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_audit_token_euid_to_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_audit_token_euid_7a32c5f8 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.audit_token.euid if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null type: string ignore_missing: true - append: + tag: append_user_effective_id_bcca5e63 field: user.effective.id value: '{{{jamf_protect.telemetry.event.btm_launch_item_remove.app.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - append: + tag: append_user_effective_id_5ce8077a field: user.effective.id value: '{{{jamf_protect.telemetry.event.btm_launch_item_remove.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.instigator?.audit_token?.euid != null && ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_executable_sha1_to_process_hash_sha1_3b606989 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_executable_sha1_to_process_hash_sha1_10bd09f0 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.executable.sha1 target_field: process.hash.sha1 if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_is_platform_binary_to_process_platform_binary_385b5efc field: jamf_protect.telemetry.event.btm_launch_item_remove.app.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_is_platform_binary_to_process_platform_binary_3dd448c7 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.is_platform_binary target_field: process.platform_binary if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_is_es_client_to_process_endpoint_security_client_5aa8020e field: jamf_protect.telemetry.event.btm_launch_item_remove.app.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_is_es_client_to_process_endpoint_security_client_8c13f797 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.is_es_client target_field: process.endpoint_security_client if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_cdhash_to_process_hash_cdhash_0c9eeb51 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_cdhash_to_process_hash_cdhash_d0607a5e field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.cdhash target_field: process.hash.cdhash if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_executable_sha1_to_process_hash_sha1_3b606989 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_executable_sha1_to_process_hash_sha1_10bd09f0 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.executable.sha1 target_field: process.hash.sha1 if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_executable_sha256_to_process_hash_sha256_89644d61 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_executable_sha256_to_process_hash_sha256_8932b388 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.executable.sha256 target_field: process.hash.sha256 if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_tty_path_to_jamf_protect_telemetry_tty_f4ceb172 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_tty_path_to_jamf_protect_telemetry_tty_cf4a7e5f field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.tty.path if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_d524aa9d field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app?.tty != null - set: + tag: set_process_interactive_2932a5b1 field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.instigator?.tty != null && ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null - set: + tag: set_process_interactive_f3841dd8 field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app?.tty == null - set: + tag: set_process_interactive_03c78afc field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.instigator?.tty == null && ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_app_audit_token_pid_to_process_pid_10b8aebc field: jamf_protect.telemetry.event.btm_launch_item_remove.app.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_3ebe139f field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_audit_token_pid_to_process_pid_f96fc495 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.audit_token.pid target_field: process.pid type: long @@ -223,28 +267,34 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5b3cd118 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_audit_token_uuid_to_process_entity_id_c747d998 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_audit_token_uuid_to_process_entity_id_8bef678b field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.audit_token.uuid target_field: process.entity_id if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_executable_path_to_process_executable_d0802f12 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.executable.path target_field: process.executable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_executable_path_to_process_executable_435a740d field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.executable.path target_field: process.executable if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -257,67 +307,80 @@ processors: } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_signing_id_to_process_code_signature_signing_id_be2e1c25 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_signing_id_to_process_code_signature_signing_id_587533e6 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.signing_id target_field: process.code_signature.signing_id if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_team_id_to_process_code_signature_team_id_43a4a19f field: jamf_protect.telemetry.event.btm_launch_item_remove.app.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_team_id_to_process_code_signature_team_id_0da17f44 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.team_id target_field: process.code_signature.team_id if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_codesigning_flags_to_process_code_signature_flags_af5d61f2 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.codesigning_flags target_field: process.code_signature.flags ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_codesigning_flags_to_process_code_signature_flags_3a9a51d1 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.codesigning_flags target_field: process.code_signature.flags if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_parent_audit_token_uuid_to_process_parent_entity_id_31380da5 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_parent_audit_token_uuid_to_process_parent_entity_id_17764d16 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.parent_audit_token.uuid target_field: process.parent.entity_id if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_parent_audit_token_pid_to_process_parent_pid_7b667415 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_parent_audit_token_pid_to_process_parent_pid_f7b0b714 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.parent_audit_token.pid target_field: process.parent.pid if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_app_parent_audit_token_euid_to_process_parent_user_id_cac242d0 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_parent_audit_token_euid_to_process_parent_user_id_57dc5eed field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.parent_audit_token.euid target_field: process.parent.user.id if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null @@ -325,20 +388,24 @@ processors: ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_parent_audit_token_e_username_to_process_parent_user_name_75a051d7 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_parent_audit_token_e_username_to_process_parent_user_name_b48fefa6 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_app_parent_audit_token_ruid_to_process_parent_real_user_id_5fd21d64 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_fa2ae8b3 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null @@ -346,11 +413,13 @@ processors: ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_app_parent_audit_token_rgid_to_process_parent_real_group_id_cfb1771c field: jamf_protect.telemetry.event.btm_launch_item_remove.app.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_e36bd349 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null @@ -358,53 +427,63 @@ processors: ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_1a650f6d field: jamf_protect.telemetry.event.btm_launch_item_remove.app.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_6a05a67c field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_parent_audit_token_exec_path_to_process_parent_executable_f1f81c2e field: jamf_protect.telemetry.event.btm_launch_item_remove.app.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_parent_audit_token_exec_path_to_process_parent_executable_baa23b47 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_responsible_audit_token_uuid_to_process_responsible_entity_id_33a9019f field: jamf_protect.telemetry.event.btm_launch_item_remove.app.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_95ccad10 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_responsible_audit_token_pid_to_process_responsible_pid_4bc62aa3 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_responsible_audit_token_pid_to_process_responsible_pid_b252123a field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.responsible_audit_token.pid target_field: process.responsible.pid if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_app_responsible_audit_token_euid_to_process_responsible_user_id_c1c49b2e field: jamf_protect.telemetry.event.btm_launch_item_remove.app.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_responsible_audit_token_euid_to_process_responsible_user_id_0fac1d93 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.responsible_audit_token.euid target_field: process.responsible.user.id if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null @@ -412,20 +491,24 @@ processors: ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_responsible_audit_token_e_username_to_process_responsible_user_name_dae6ed6f field: jamf_protect.telemetry.event.btm_launch_item_remove.app.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_responsible_audit_token_e_username_to_process_responsible_user_name_06e6bd5e field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_app_responsible_audit_token_ruid_to_process_responsible_real_user_id_36618524 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_27855263 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null @@ -433,11 +516,13 @@ processors: ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_app_responsible_audit_token_rgid_to_process_responsible_real_group_id_565a0250 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_2ddcba71 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id if: ctx.jamf_protect.telemetry?.event?.btm_launch_item_remove?.app == null @@ -445,23 +530,28 @@ processors: ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_bd95d26b field: jamf_protect.telemetry.event.btm_launch_item_remove.app.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_8bda3166 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_app_responsible_audit_token_exec_path_to_process_responsible_executable_d2a66750 field: jamf_protect.telemetry.event.btm_launch_item_remove.app.responsible_audit_token.exec_path target_field: process.responsible.executable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_btm_launch_item_remove_instigator_responsible_audit_token_exec_path_to_process_responsible_executable_5029e875 field: jamf_protect.telemetry.event.btm_launch_item_remove.instigator.responsible_audit_token.exec_path target_field: process.responsible.executable ignore_missing: true - pipeline: + tag: pipeline_ee2665c0 name: '{{ IngestPipeline "pipeline_object_process" }}' if: ctx.jamf_protect?.telemetry?.event?.btm_launch_item_remove?.instigator == null && ctx.jamf_protect?.telemetry?.event?.btm_launch_item_remove?.app == null diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_chroot.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_chroot.yml index ad7fbe8abe7..7871dbb1d90 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_chroot.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_chroot.yml @@ -2,21 +2,25 @@ description: Pipeline for parsing specific fields related to chroot events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_038dee41 field: event.reason value: Software has changed its apparent root directory in which it's actively operating out of - rename: + tag: rename_jamf_protect_telemetry_event_chroot_target_path_to_file_path_09b1eb15 field: jamf_protect.telemetry.event.chroot.target.path target_field: file.path ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - append: + tag: append_event_type_0602e5be field: event.type value: change - pipeline: + tag: pipeline_0d75fa7a name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_cs_invalidated.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_cs_invalidated.yml index a515f343db6..2acf85a8152 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_cs_invalidated.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_cs_invalidated.yml @@ -2,14 +2,16 @@ description: Pipeline for parsing specific fields related to codesignature invalid events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_48b0a734 field: event.reason value: The system detected that a process has had its code signature marked as invalid -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - pipeline: + tag: pipeline_0d75fa7a name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_exec.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_exec.yml index 1f25f69fbe2..772c494e283 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_exec.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_exec.yml @@ -2,112 +2,136 @@ description: Pipeline for parsing specific fields related to exec events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_3b9209af field: event.reason value: A new process has been executed - append: + tag: append_event_type_fffb7a5a field: event.type value: start - rename: + tag: rename_jamf_protect_telemetry_event_exec_script_path_to_file_path_9cdfb881 field: jamf_protect.telemetry.event.exec.script.path target_field: file.path ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_start_time_to_process_start_a622f76c field: jamf_protect.telemetry.event.exec.target.start_time target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_exec_target_audit_token_egid_to_jamf_protect_telemetry_event_exec_target_audit_token_egid_6673124a field: jamf_protect.telemetry.event.exec.target.audit_token.egid target_field: jamf_protect.telemetry.event.exec.target.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_exec_target_audit_token_euid_to_jamf_protect_telemetry_event_exec_target_audit_token_euid_8487701e field: jamf_protect.telemetry.event.exec.target.audit_token.euid target_field: jamf_protect.telemetry.event.exec.target.audit_token.euid type: string ignore_missing: true - append: + tag: append_user_effective_id_4ec8cc8f field: user.effective.id value: '{{{jamf_protect.telemetry.event.exec.target.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.exec?.target?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_audit_token_e_username_to_process_user_name_d80605ed field: jamf_protect.telemetry.event.exec.target.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_is_platform_binary_to_process_platform_binary_42dc9cb1 field: jamf_protect.telemetry.event.exec.target.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_is_es_client_to_process_endpoint_security_client_a1fdb211 field: jamf_protect.telemetry.event.exec.target.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_cdhash_to_process_hash_cdhash_b622bd14 field: jamf_protect.telemetry.event.exec.target.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_executable_sha1_to_process_hash_sha1_246c2aca field: jamf_protect.telemetry.event.exec.target.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_executable_sha256_to_process_hash_sha256_9d3556a2 field: jamf_protect.telemetry.event.exec.target.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_tty_path_to_jamf_protect_telemetry_tty_2e81f66d field: jamf_protect.telemetry.event.exec.target.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_959b36ce field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.exec?.target?.tty != null - set: + tag: set_process_interactive_e60fa981 field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.exec?.target?.tty == null - convert: + tag: convert_jamf_protect_telemetry_event_exec_target_audit_token_pid_to_process_pid_52fb8f6f field: jamf_protect.telemetry.event.exec.target.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_cb1421f6 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_audit_token_uuid_to_process_entity_id_ac1e0569 field: jamf_protect.telemetry.event.exec.target.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_executable_path_to_process_executable_46d78b57 field: jamf_protect.telemetry.event.exec.target.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -119,16 +143,19 @@ processors: } } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_exec_args_to_process_args_dfd25ea5 field: jamf_protect.telemetry.event.exec.args target_field: process.args ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.exec?.args != null - script: + tag: script_c76fe348 description: Counts the indexes of the arguments lang: painless source: > @@ -140,103 +167,124 @@ processors: } ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_exec_env_to_process_env_vars_ea9ed1e2 field: jamf_protect.telemetry.event.exec.env target_field: process.env_vars ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.exec?.env != null - rename: + tag: rename_jamf_protect_telemetry_event_exec_cwd_path_to_process_working_directory_45e32ab4 field: jamf_protect.telemetry.event.exec.cwd.path target_field: process.working_directory ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.exec?.cwd?.path != null - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_signing_id_to_process_code_signature_signing_id_c120f59c field: jamf_protect.telemetry.event.exec.target.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_team_id_to_process_code_signature_team_id_6e8fc102 field: jamf_protect.telemetry.event.exec.target.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_codesigning_flags_to_process_code_signature_flags_9897fe87 field: jamf_protect.telemetry.event.exec.target.codesigning_flags target_field: process.code_signature.flags ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_parent_audit_token_uuid_to_process_parent_entity_id_49f6450c field: jamf_protect.telemetry.event.exec.target.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_parent_audit_token_pid_to_process_parent_pid_e3a39a1a field: jamf_protect.telemetry.event.exec.target.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_exec_target_parent_audit_token_euid_to_process_parent_user_id_7f3b8d4b field: jamf_protect.telemetry.event.exec.target.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_exec_target_parent_audit_token_ruid_to_process_parent_real_user_id_122ad61d field: jamf_protect.telemetry.event.exec.target.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_parent_audit_token_e_username_to_process_parent_user_name_abed614e field: jamf_protect.telemetry.event.exec.target.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_exec_target_parent_audit_token_rgid_to_process_parent_real_group_id_d0c56d4b field: jamf_protect.telemetry.event.exec.target.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_0b07ee14 field: jamf_protect.telemetry.event.exec.target.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_parent_audit_token_exec_path_to_process_parent_executable_72ef8f1f field: jamf_protect.telemetry.event.exec.target.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_responsible_audit_token_uuid_to_process_responsible_entity_id_83e3c7c2 field: jamf_protect.telemetry.event.exec.target.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_responsible_audit_token_pid_to_process_responsible_pid_ef7b848c field: jamf_protect.telemetry.event.exec.target.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_exec_target_responsible_audit_token_euid_to_process_responsible_user_id_a0164bd5 field: jamf_protect.telemetry.event.exec.target.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_responsible_audit_token_e_username_to_process_responsible_user_name_1168dc66 field: jamf_protect.telemetry.event.exec.target.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_exec_target_responsible_audit_token_ruid_to_process_responsible_real_user_id_5f848b99 field: jamf_protect.telemetry.event.exec.target.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_exec_target_responsible_audit_token_rgid_to_process_responsible_real_group_id_adcae0af field: jamf_protect.telemetry.event.exec.target.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_4cb1fc9e field: jamf_protect.telemetry.event.exec.target.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_exec_target_responsible_audit_token_exec_path_to_process_responsible_executable_9adc1c0d field: jamf_protect.telemetry.event.exec.target.responsible_audit_token.exec_path target_field: process.responsible.executable - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_file_collection.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_file_collection.yml index 91a1bd62070..ef1cb135169 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_file_collection.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_file_collection.yml @@ -2,17 +2,20 @@ description: Pipeline for Jamf Protect Telemetry crash and diagnostic events. processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_549da753 field: event.reason value: A crash or diagnostic file is detected being created - rename: + tag: rename_jamf_protect_telemetry_event_file_collection_file_path_to_log_file_path_aed6a940 field: jamf_protect.telemetry.event.file_collection.file.path target_field: log.file.path ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_file_collection_contents_to_jamf_protect_telemetry_log_entries_28a2427e field: jamf_protect.telemetry.event.file_collection.contents target_field: jamf_protect.telemetry.log_entries type: string diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_gatekeeper_user_override.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_gatekeeper_user_override.yml index f95768f2567..eb70a4130c6 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_gatekeeper_user_override.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_gatekeeper_user_override.yml @@ -2,70 +2,85 @@ description: Pipeline for parsing specific fields related to GateKeeper events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_a9f28ff4 field: event.reason value: Gatekeeper controls were overridden - append: + tag: append_event_category_557b9332 field: event.category value: malware allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_gatekeeper_user_override_file_path_to_file_path_1213b77d field: jamf_protect.telemetry.event.gatekeeper_user_override.file.path target_field: file.path ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_gatekeeper_user_override_file_stat_st_size_to_file_size_2207a759 field: jamf_protect.telemetry.event.gatekeeper_user_override.file.stat.st_size target_field: file.size ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_gatekeeper_user_override_file_stat_st_birthtimespec_to_file_created_61d99d8c field: jamf_protect.telemetry.event.gatekeeper_user_override.file.stat.st_birthtimespec target_field: file.created ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_gatekeeper_user_override_file_stat_st_atimespec_to_file_accessed_d2ab8e4b field: jamf_protect.telemetry.event.gatekeeper_user_override.file.stat.st_atimespec target_field: file.accessed ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_gatekeeper_user_override_file_stat_st_uid_to_file_uid_c0dd826f field: jamf_protect.telemetry.event.gatekeeper_user_override.file.stat.st_uid target_field: file.uid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_gatekeeper_user_override_file_stat_st_gid_to_file_gid_69c0d2b7 field: jamf_protect.telemetry.event.gatekeeper_user_override.file.stat.st_gid target_field: file.gid type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_gatekeeper_user_override_sha256_to_file_hash_sha256_7551d278 field: jamf_protect.telemetry.event.gatekeeper_user_override.sha256 target_field: file.hash.sha256 if: ctx.jamf_protect?.telemetry?.event?.gatekeeper_user_override?.sha256 != "NULL" ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_gatekeeper_user_override_signing_info_cdhash_to_file_hash_cdhash_837ce2d5 field: jamf_protect.telemetry.event.gatekeeper_user_override.signing_info.cdhash target_field: file.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_gatekeeper_user_override_signing_info_team_id_to_file_code_signature_team_id_1f7460f9 field: jamf_protect.telemetry.event.gatekeeper_user_override.signing_info.team_id target_field: file.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_gatekeeper_user_override_signing_info_signing_id_to_file_code_signature_signing_id_20b644d5 field: jamf_protect.telemetry.event.gatekeeper_user_override.signing_info.signing_id target_field: file.code_signature.signing_id ignore_missing: true - append: + tag: append_threat_indicator_file_path_88c08c7f field: threat.indicator.file.path value: '{{{file.path}}}' if: ctx.file?.path != null && ctx.file.path != '' allow_duplicates: false - set: + tag: set_threat_indicator_type_e912359f field: threat.indicator.type value: file - -########################## -## ECS Process ## -########################## + + ########################## + ## ECS Process ## + ########################## - pipeline: + tag: pipeline_0d75fa7a name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_kextload.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_kextload.yml index 9c8a47a9b46..e8daf834826 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_kextload.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_kextload.yml @@ -2,25 +2,30 @@ description: Pipeline for parsing specific fields related to kext load events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_22047e84 field: event.reason value: A kernel extension (kext) was loaded - convert: + tag: convert_jamf_protect_telemetry_event_kextload_identifier_to_jamf_protect_telemetry_identifier_c237e93a field: jamf_protect.telemetry.event.kextload.identifier target_field: jamf_protect.telemetry.identifier type: string ignore_missing: true - append: + tag: append_event_type_0602e5be field: event.type value: change - append: + tag: append_event_category_1c667e2c field: event.category - value: configuration -########################## -## ECS Process ## -########################## + value: configuration + ########################## + ## ECS Process ## + ########################## - pipeline: + tag: pipeline_0d75fa7a name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_kextunload.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_kextunload.yml index 40636334b31..e2a5aed0d94 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_kextunload.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_kextunload.yml @@ -2,25 +2,30 @@ description: Pipeline for parsing specific fields related to kext unload events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_cf03dec5 field: event.reason value: A kernel extension (kext) was unloaded - convert: + tag: convert_jamf_protect_telemetry_event_kextunload_identifier_to_jamf_protect_telemetry_identifier_a8463343 field: jamf_protect.telemetry.event.kextunload.identifier target_field: jamf_protect.telemetry.identifier type: string ignore_missing: true - append: + tag: append_event_type_0602e5be field: event.type value: change - append: + tag: append_event_category_1c667e2c field: event.category - value: configuration -########################## -## ECS Process ## -########################## + value: configuration + ########################## + ## ECS Process ## + ########################## - pipeline: + tag: pipeline_0d75fa7a name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_log_collection.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_log_collection.yml index e2ee75833f4..2bcac3880cc 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_log_collection.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_log_collection.yml @@ -2,18 +2,21 @@ description: Pipeline for Jamf Protect Telemetry log collection events. processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_3c1b91ae field: event.reason value: New entries have been collected from a log file - rename: + tag: rename_jamf_protect_telemetry_event_log_collection_path_to_log_file_path_db8f7312 field: jamf_protect.telemetry.event.log_collection.path target_field: log.file.path ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_log_collection_texts_to_jamf_protect_telemetry_log_entries_36d5b2be field: jamf_protect.telemetry.event.log_collection.texts target_field: jamf_protect.telemetry.log_entries type: string - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_login_login.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_login_login.yml index fac7768ec0d..ca13ea4f750 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_login_login.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_login_login.yml @@ -2,19 +2,23 @@ description: Pipeline for parsing specific fields related to login using /usr/bin/login events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_5f73a905 field: event.reason value: A user attempted to log in using /usr/bin/login - append: + tag: append_event_type_fffb7a5a field: event.type value: start - append: + tag: append_event_category_109b4d72 field: event.category - value: authentication + value: authentication - script: + tag: script_859f908d lang: painless source: > ctx.event = ctx.event != null ? ctx.event : new HashMap(); @@ -29,16 +33,19 @@ processors: ctx.event.outcome = 'unknown'; } - rename: + tag: rename_jamf_protect_telemetry_event_login_login_username_to_user_name_091a3af8 field: jamf_protect.telemetry.event.login_login.username target_field: user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_login_login_uid_to_user_id_741c8aae field: jamf_protect.telemetry.event.login_login.uid target_field: user.id type: string ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - pipeline: + tag: pipeline_0d75fa7a name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_login_logout.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_login_logout.yml index 04f4a2731a4..a60c8c31372 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_login_logout.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_login_logout.yml @@ -2,29 +2,35 @@ description: Pipeline for parsing specific fields related to logout using /usr/bin/logout events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_f2c95969 field: event.reason value: A user logged out from /usr/bin/login - append: + tag: append_event_type_90fe2b35 field: event.type value: end - append: + tag: append_event_category_109b4d72 field: event.category - value: authentication + value: authentication - rename: + tag: rename_jamf_protect_telemetry_event_login_logout_username_to_user_name_a2cc0791 field: jamf_protect.telemetry.event.login_logout.username target_field: user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_login_logout_uid_to_user_id_8f3f384b field: jamf_protect.telemetry.event.login_logout.uid target_field: user.id type: string ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - pipeline: + tag: pipeline_0d75fa7a name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_lw_session_lock.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_lw_session_lock.yml index 00ed0f8ed91..3ec0da68e6b 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_lw_session_lock.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_lw_session_lock.yml @@ -2,29 +2,35 @@ description: Pipeline for parsing specific fields related to loginwindow lock events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_9aa8f879 field: event.reason value: A user has locked the screen - append: + tag: append_event_type_fffb7a5a field: event.type value: start - append: + tag: append_event_category_109b4d72 field: event.category - value: authentication + value: authentication - convert: + tag: convert_jamf_protect_telemetry_event_lw_session_lock_graphical_session_id_to_jamf_protect_telemetry_graphical_authentication_username_7515cd33 field: jamf_protect.telemetry.event.lw_session_lock.graphical_session_id target_field: jamf_protect.telemetry.graphical_authentication_username type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_lw_session_lock_username_to_user_name_d2d65a09 field: jamf_protect.telemetry.event.lw_session_lock.username target_field: user.name ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - pipeline: + tag: pipeline_0d75fa7a name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_lw_session_login.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_lw_session_login.yml index 30af818c7cf..dba4c707493 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_lw_session_login.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_lw_session_login.yml @@ -2,29 +2,35 @@ description: Pipeline for parsing specific fields related to loginwindow login events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_2108fe53 field: event.reason value: A user has logged in via the Login Window - append: + tag: append_event_type_fffb7a5a field: event.type value: start - append: + tag: append_event_category_109b4d72 field: event.category - value: authentication + value: authentication - convert: + tag: convert_jamf_protect_telemetry_event_lw_session_login_graphical_session_id_to_jamf_protect_telemetry_graphical_authentication_username_2559b391 field: jamf_protect.telemetry.event.lw_session_login.graphical_session_id target_field: jamf_protect.telemetry.graphical_authentication_username type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_lw_session_login_username_to_user_name_b56d7299 field: jamf_protect.telemetry.event.lw_session_login.username target_field: user.name ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - pipeline: + tag: pipeline_0d75fa7a name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_lw_session_logout.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_lw_session_logout.yml index d5ee23ea5a4..88c1ff29c01 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_lw_session_logout.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_lw_session_logout.yml @@ -2,29 +2,35 @@ description: Pipeline for parsing specific fields related to loginwindow logout events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_948f99bf field: event.reason value: A user has logged out of an active graphical session - append: + tag: append_event_type_90fe2b35 field: event.type value: end - append: + tag: append_event_category_109b4d72 field: event.category - value: authentication + value: authentication - convert: + tag: convert_jamf_protect_telemetry_event_lw_session_logout_graphical_session_id_to_jamf_protect_telemetry_graphical_authentication_username_b5e77eea field: jamf_protect.telemetry.event.lw_session_logout.graphical_session_id target_field: jamf_protect.telemetry.graphical_authentication_username type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_lw_session_logout_username_to_user_name_a2480ffa field: jamf_protect.telemetry.event.lw_session_logout.username target_field: user.name ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - pipeline: + tag: pipeline_0d75fa7a name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_lw_session_unlock.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_lw_session_unlock.yml index 04ad4ef6e9b..f219aeb3a72 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_lw_session_unlock.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_lw_session_unlock.yml @@ -2,29 +2,35 @@ description: Pipeline for parsing specific fields related to loginwindow unloack events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_e047e9c4 field: event.reason value: A user has unlocked the screen from the Login Window - append: + tag: append_event_type_fffb7a5a field: event.type value: start - append: + tag: append_event_category_109b4d72 field: event.category - value: authentication + value: authentication - convert: + tag: convert_jamf_protect_telemetry_event_lw_session_unlock_graphical_session_id_to_jamf_protect_telemetry_graphical_authentication_username_0d476518 field: jamf_protect.telemetry.event.lw_session_unlock.graphical_session_id target_field: jamf_protect.telemetry.graphical_authentication_username type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_lw_session_unlock_username_to_user_name_bfd8916c field: jamf_protect.telemetry.event.lw_session_unlock.username target_field: user.name ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - pipeline: + tag: pipeline_0d75fa7a name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_mount.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_mount.yml index a98a5c00a90..477dac519a1 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_mount.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_mount.yml @@ -2,69 +2,85 @@ description: Pipeline for parsing specific fields related to volumen mount events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_a4ad5cac field: event.reason value: A file system has been mounted - append: + tag: append_event_type_fffb7a5a field: event.type value: start - rename: + tag: rename_jamf_protect_telemetry_event_mount_statfs_f_mntfromname_to_volume_device_name_3635b1c2 field: jamf_protect.telemetry.event.mount.statfs.f_mntfromname target_field: volume.device_name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_mount_statfs_f_mntonname_to_volume_mount_name_31270bf4 field: jamf_protect.telemetry.event.mount.statfs.f_mntonname target_field: volume.mount_name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_mount_statfs_f_fstypename_to_volume_file_system_type_8ecedb67 field: jamf_protect.telemetry.event.mount.statfs.f_fstypename target_field: volume.file_system_type ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_mount_device_size_to_volume_size_d9d8973e field: jamf_protect.telemetry.event.mount.device.size target_field: volume.size ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_mount_device_bus_name_to_volume_bus_type_3e721149 field: jamf_protect.telemetry.event.mount.device.bus_name target_field: volume.bus_type ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_mount_device_product_id_to_volume_product_id_c1d0594e field: jamf_protect.telemetry.event.mount.device.product_id target_field: volume.product_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_mount_device_product_name_to_volume_product_name_2714306a field: jamf_protect.telemetry.event.mount.device.product_name target_field: volume.product_name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_mount_device_vendor_id_to_volume_vendor_id_34c0d626 field: jamf_protect.telemetry.event.mount.device.vendor_id target_field: volume.vendor_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_mount_device_vendor_name_to_volume_vendor_name_269ec35a field: jamf_protect.telemetry.event.mount.device.vendor_name target_field: volume.vendor_name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_mount_device_protocol_to_volume_device_type_2e908941 field: jamf_protect.telemetry.event.mount.device.protocol target_field: volume.device_type ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_mount_device_is_removable_to_volume_removable_7d95c9bd field: jamf_protect.telemetry.event.mount.device.is_removable target_field: volume.removable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_mount_device_is_writable_to_volume_writable_e0889b67 field: jamf_protect.telemetry.event.mount.device.is_writable target_field: volume.writable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_mount_device_serial_number_to_volume_serial_number_278cbd66 field: jamf_protect.telemetry.event.mount.device.serial_number target_field: volume.serial_number ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - pipeline: - name: '{{ IngestPipeline "pipeline_object_process" }}' \ No newline at end of file + tag: pipeline_0d75fa7a + name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_network_connect.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_network_connect.yml index 1b6f6371f70..6f1f71845f8 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_network_connect.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_network_connect.yml @@ -2,47 +2,57 @@ description: Pipeline for parsing specific fields related to Network Connection events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_ae8f5049 field: event.reason value: A network connection has been established - append: + tag: append_event_category_7afdca3c field: event.category - value: network + value: network - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_file_direction_to_network_direction_4ca00dca field: jamf_protect.telemetry.event.network_connect.file.direction target_field: network.direction ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_remote_hostname_to_destination_domain_dc8eb992 field: jamf_protect.telemetry.event.network_connect.remote_hostname target_field: destination.domain ignore_missing: true - append: + tag: append_url_domain_f06bd042 field: url.domain value: '{{{destination.domain}}}' if: ctx.destination?.domain != null allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_remote_endpoint_host_to_destination_ip_aeb2225f field: jamf_protect.telemetry.event.network_connect.remote_endpoint.host target_field: destination.ip ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_network_connect_remote_endpoint_port_to_destination_port_061267b1 field: jamf_protect.telemetry.event.network_connect.remote_endpoint.port target_field: destination.port type: integer ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_local_endpoint_host_to_source_ip_b31fad8d field: jamf_protect.telemetry.event.network_connect.local_endpoint.host target_field: source.ip ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_network_connect_local_endpoint_port_to_source_port_209340d9 field: jamf_protect.telemetry.event.network_connect.local_endpoint.port target_field: source.port type: integer ignore_missing: true - script: + tag: script_3832285b lang: painless params: itemTypeMap: @@ -51,7 +61,7 @@ processors: '17': udp '41': ip6in4 '58': icmp6 - '255': ipproto_raw + '255': ipproto_raw source: > if (ctx.jamf_protect?.telemetry?.event?.network_connect?.socket_protocol != null) { String itemType = ctx.jamf_protect.telemetry.event.network_connect.socket_protocol.toString(); @@ -64,6 +74,7 @@ processors: ctx.network.transport = itemTypeString; } - script: + tag: script_a3b43c95 lang: painless params: itemTypeMap: @@ -78,6 +89,7 @@ processors: ctx.jamf_protect.telemetry.network_socket_family = itemTypeString; } - script: + tag: script_880b881e lang: painless params: itemTypeMap: @@ -93,98 +105,119 @@ processors: ctx.jamf_protect = ctx.jamf_protect != null ? ctx.jamf_protect : new HashMap(); ctx.jamf_protect.telemetry.network_socket_type = itemTypeString; } -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_start_time_to_process_start_2b3435d5 field: jamf_protect.telemetry.event.network_connect.instigator.start_time target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_network_connect_instigator_audit_token_egid_to_jamf_protect_telemetry_event_network_connect_instigator_audit_token_egid_26e2ca76 field: jamf_protect.telemetry.event.network_connect.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.network_connect.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_network_connect_instigator_audit_token_euid_to_jamf_protect_telemetry_event_network_connect_instigator_audit_token_euid_d036039a field: jamf_protect.telemetry.event.network_connect.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.network_connect.instigator.audit_token.euid type: string ignore_missing: true - append: + tag: append_user_effective_id_0780bf03 field: user.effective.id value: '{{{jamf_protect.telemetry.event.network_connect.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.network_connect?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_audit_token_e_username_to_process_user_name_23044fcc field: jamf_protect.telemetry.event.network_connect.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_is_platform_binary_to_process_platform_binary_2a1b7f88 field: jamf_protect.telemetry.event.network_connect.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_is_es_client_to_process_endpoint_security_client_0088362a field: jamf_protect.telemetry.event.network_connect.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_cdhash_to_process_hash_cdhash_d0b376dd field: jamf_protect.telemetry.event.network_connect.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_executable_sha1_to_process_hash_sha1_939563f5 field: jamf_protect.telemetry.event.network_connect.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_executable_sha256_to_process_hash_sha256_64387f6d field: jamf_protect.telemetry.event.network_connect.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_tty_path_to_jamf_protect_telemetry_tty_5d008a86 field: jamf_protect.telemetry.event.network_connect.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_ac02fccd field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.network_connect?.instigator?.tty != null - set: + tag: set_process_interactive_748ec0c8 field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.network_connect?.instigator?.tty == null - convert: + tag: convert_jamf_protect_telemetry_event_network_connect_instigator_audit_token_pid_to_process_pid_42e77bd8 field: jamf_protect.telemetry.event.network_connect.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_40a1108f field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_audit_token_uuid_to_process_entity_id_aace0a1c field: jamf_protect.telemetry.event.network_connect.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_executable_path_to_process_executable_f45851c6 field: jamf_protect.telemetry.event.network_connect.instigator.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -196,104 +229,124 @@ processors: } } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_db_path_to_process_working_directory_2e5149cd field: jamf_protect.telemetry.event.network_connect.db_path target_field: process.working_directory ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.network_connect?.db_path != null - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_signing_id_to_process_code_signature_signing_id_a77ebe79 field: jamf_protect.telemetry.event.network_connect.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_team_id_to_process_code_signature_team_id_907ecea3 field: jamf_protect.telemetry.event.network_connect.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_codesigning_flags_to_process_code_signature_flags_071b4d26 field: jamf_protect.telemetry.event.network_connect.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_parent_audit_token_uuid_to_process_parent_entity_id_d81e1dd9 field: jamf_protect.telemetry.event.network_connect.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_parent_audit_token_pid_to_process_parent_pid_434781f9 field: jamf_protect.telemetry.event.network_connect.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_network_connect_instigator_parent_audit_token_euid_to_process_parent_user_id_350cc904 field: jamf_protect.telemetry.event.network_connect.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_parent_audit_token_e_username_to_process_parent_user_name_7c4cad4b field: jamf_protect.telemetry.event.network_connect.instigator.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_network_connect_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_685ed590 field: jamf_protect.telemetry.event.network_connect.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_network_connect_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_ac8db528 field: jamf_protect.telemetry.event.network_connect.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_898c7e19 field: jamf_protect.telemetry.event.network_connect.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_parent_audit_token_exec_path_to_process_parent_executable_b39c4fba field: jamf_protect.telemetry.event.network_connect.instigator.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## - + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_edec8793 field: jamf_protect.telemetry.event.network_connect.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_responsible_audit_token_pid_to_process_responsible_pid_d3547977 field: jamf_protect.telemetry.event.network_connect.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_network_connect_instigator_responsible_audit_token_euid_to_process_responsible_user_id_408691e2 field: jamf_protect.telemetry.event.network_connect.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_responsible_audit_token_e_username_to_process_responsible_user_name_096833b3 field: jamf_protect.telemetry.event.network_connect.instigator.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_network_connect_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_1fd01af0 field: jamf_protect.telemetry.event.network_connect.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_network_connect_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_45953c2c field: jamf_protect.telemetry.event.network_connect.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_c0db8c27 field: jamf_protect.telemetry.event.network_connect.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_network_connect_instigator_responsible_audit_token_exec_path_to_process_responsible_executable_6049a94c field: jamf_protect.telemetry.event.network_connect.instigator.responsible_audit_token.exec_path target_field: process.responsible.executable - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_attribute_set.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_attribute_set.yml index 2356fcd5276..941d903cc58 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_attribute_set.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_attribute_set.yml @@ -2,133 +2,162 @@ description: Pipeline for parsing specific fields related to attribute set events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_0699beb1 field: event.reason value: Attribute set on user or group using Open Directory - append: + tag: append_event_type_fbbdb681 field: event.type value: creation - append: + tag: append_event_category_1c667e2c field: event.category - value: configuration + value: configuration - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_group_name_to_group_name_bf39c915 field: jamf_protect.telemetry.event.od_attribute_set.group_name target_field: group.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_db_path_to_file_path_5f80d53b field: jamf_protect.telemetry.event.od_attribute_set.db_path target_field: file.path ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_set_record_name_to_jamf_protect_telemetry_record_name_96670b3d field: jamf_protect.telemetry.event.od_attribute_set.record_name target_field: jamf_protect.telemetry.record_name type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_set_attribute_name_to_jamf_protect_telemetry_attribute_name_5780b48d field: jamf_protect.telemetry.event.od_attribute_set.attribute_name target_field: jamf_protect.telemetry.attribute_name type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_set_attribute_value_to_jamf_protect_telemetry_attribute_value_28bb5311 field: jamf_protect.telemetry.event.od_attribute_set.attribute_value target_field: jamf_protect.telemetry.attribute_value type: string ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_start_time_to_process_start_0103938d field: jamf_protect.telemetry.event.od_attribute_set.instigator.start_time target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_set_instigator_audit_token_egid_to_jamf_protect_telemetry_event_od_attribute_set_instigator_audit_token_egid_5546b604 field: jamf_protect.telemetry.event.od_attribute_set.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.od_attribute_set.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_set_instigator_audit_token_euid_to_jamf_protect_telemetry_event_od_attribute_set_instigator_audit_token_euid_74ed8750 field: jamf_protect.telemetry.event.od_attribute_set.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.od_attribute_set.instigator.audit_token.euid type: string ignore_missing: true - append: + tag: append_user_effective_id_469620dd field: user.effective.id value: '{{{jamf_protect.telemetry.event.od_attribute_set.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.od_attribute_set?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_audit_token_e_username_to_process_user_name_e029eed4 field: jamf_protect.telemetry.event.od_attribute_set.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_is_platform_binary_to_process_platform_binary_8a2795b0 field: jamf_protect.telemetry.event.od_attribute_set.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_is_es_client_to_process_endpoint_security_client_42403572 field: jamf_protect.telemetry.event.od_attribute_set.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_cdhash_to_process_hash_cdhash_02bf9df5 field: jamf_protect.telemetry.event.od_attribute_set.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_executable_sha1_to_process_hash_sha1_bfae892d field: jamf_protect.telemetry.event.od_attribute_set.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_executable_sha256_to_process_hash_sha256_a35b27c5 field: jamf_protect.telemetry.event.od_attribute_set.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_tty_path_to_jamf_protect_telemetry_tty_b53414ee field: jamf_protect.telemetry.event.od_attribute_set.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_98930807 field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.od_attribute_set?.instigator?.tty != null - set: + tag: set_process_interactive_d484e37a field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.od_attribute_set?.instigator?.tty == null - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_set_instigator_audit_token_pid_to_process_pid_289a772a field: jamf_protect.telemetry.event.od_attribute_set.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_c6ae2f7f field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_audit_token_uuid_to_process_entity_id_c5aaa104 field: jamf_protect.telemetry.event.od_attribute_set.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_executable_path_to_process_executable_2127034e field: jamf_protect.telemetry.event.od_attribute_set.instigator.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -140,103 +169,124 @@ processors: } } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_db_path_to_process_working_directory_a696d77b field: jamf_protect.telemetry.event.od_attribute_set.db_path target_field: process.working_directory ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.od_attribute_set?.db_path != null - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_signing_id_to_process_code_signature_signing_id_e060e231 field: jamf_protect.telemetry.event.od_attribute_set.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_team_id_to_process_code_signature_team_id_eeb498fb field: jamf_protect.telemetry.event.od_attribute_set.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_set_instigator_codesigning_flags_to_process_code_signature_flags_b15364c8 field: jamf_protect.telemetry.event.od_attribute_value_set.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_parent_audit_token_uuid_to_process_parent_entity_id_f4e9d001 field: jamf_protect.telemetry.event.od_attribute_set.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_parent_audit_token_pid_to_process_parent_pid_0ee9f271 field: jamf_protect.telemetry.event.od_attribute_set.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_set_instigator_parent_audit_token_euid_to_process_parent_user_id_1a21d746 field: jamf_protect.telemetry.event.od_attribute_set.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_parent_audit_token_e_username_to_process_parent_user_name_9dc24b13 field: jamf_protect.telemetry.event.od_attribute_set.instigator.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_set_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_5e8048fa field: jamf_protect.telemetry.event.od_attribute_set.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_set_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_716b8416 field: jamf_protect.telemetry.event.od_attribute_set.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_de616211 field: jamf_protect.telemetry.event.od_attribute_set.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_parent_audit_token_exec_path_to_process_parent_executable_ebf8d372 field: jamf_protect.telemetry.event.od_attribute_set.instigator.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_0a26081b field: jamf_protect.telemetry.event.od_attribute_set.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_responsible_audit_token_pid_to_process_responsible_pid_c637096f field: jamf_protect.telemetry.event.od_attribute_set.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_set_instigator_responsible_audit_token_euid_to_process_responsible_user_id_c8056784 field: jamf_protect.telemetry.event.od_attribute_set.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_responsible_audit_token_e_username_to_process_responsible_user_name_c6c265bb field: jamf_protect.telemetry.event.od_attribute_set.instigator.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_set_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_81612832 field: jamf_protect.telemetry.event.od_attribute_set.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_set_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_bb0af39a field: jamf_protect.telemetry.event.od_attribute_set.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_3502ef7f field: jamf_protect.telemetry.event.od_attribute_set.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_set_instigator_responsible_audit_token_exec_path_to_process_responsible_executable_95857964 field: jamf_protect.telemetry.event.od_attribute_set.instigator.responsible_audit_token.exec_path target_field: process.responsible.executable ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_attribute_value_add.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_attribute_value_add.yml index 1baa1559060..3f21a32ceb3 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_attribute_value_add.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_attribute_value_add.yml @@ -2,132 +2,161 @@ description: Pipeline for parsing specific fields related to attribute value add events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_870b2664 field: event.reason value: Attribute added to a user or group using Open Directory - append: + tag: append_event_type_fbbdb681 field: event.type value: creation - append: + tag: append_event_category_1c667e2c field: event.category value: configuration - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_group_name_to_group_name_ec2dfde4 field: jamf_protect.telemetry.event.od_attribute_value_add.group_name target_field: group.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_record_name_to_user_name_a8aa4726 field: jamf_protect.telemetry.event.od_attribute_value_add.record_name target_field: user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_db_path_to_file_path_04893f72 field: jamf_protect.telemetry.event.od_attribute_value_add.db_path target_field: file.path ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_add_attribute_name_to_jamf_protect_telemetry_attribute_name_8677a06e field: jamf_protect.telemetry.event.od_attribute_value_add.attribute_name target_field: jamf_protect.telemetry.attribute_name type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_add_attribute_value_to_jamf_protect_telemetry_attribute_value_ca594f7e field: jamf_protect.telemetry.event.od_attribute_value_add.attribute_value target_field: jamf_protect.telemetry.attribute_value type: string ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_start_time_to_process_start_317a3d58 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.start_time target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_add_instigator_audit_token_egid_to_jamf_protect_telemetry_event_od_attribute_value_add_instigator_audit_token_egid_b2a530fe field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_add_instigator_audit_token_euid_to_jamf_protect_telemetry_event_od_attribute_value_add_instigator_audit_token_euid_057790ee field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.audit_token.euid type: string ignore_missing: true - append: + tag: append_user_effective_id_95fbb7c1 field: user.effective.id value: '{{{jamf_protect.telemetry.event.od_attribute_value_add.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.od_attribute_value_add?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_audit_token_e_username_to_process_user_name_f2046ab9 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_is_platform_binary_to_process_platform_binary_63301c55 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_is_es_client_to_process_endpoint_security_client_d88b6025 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_cdhash_to_process_hash_cdhash_b26c7538 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_executable_sha1_to_process_hash_sha1_b7efa84e field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_executable_sha256_to_process_hash_sha256_ea6a3586 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_tty_path_to_jamf_protect_telemetry_tty_eb0a3be9 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_2d4595a8 field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.od_attribute_value_add?.instigator?.tty != null - set: + tag: set_process_interactive_2020aab3 field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.od_attribute_value_add?.instigator?.tty == null - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_add_instigator_audit_token_pid_to_process_pid_d81764ef field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_72c1a5d6 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_audit_token_uuid_to_process_entity_id_07e99f35 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_executable_path_to_process_executable_b7a65db3 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -139,102 +168,123 @@ processors: } } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_db_path_to_process_working_directory_4b6f5c17 field: jamf_protect.telemetry.event.od_attribute_value_add.db_path target_field: process.working_directory ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.od_attribute_value_add?.db_path != null - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_signing_id_to_process_code_signature_signing_id_06ae8228 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_team_id_to_process_code_signature_team_id_7522c27e field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_codesigning_flags_to_process_code_signature_flags_1afc07f3 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_parent_audit_token_uuid_to_process_parent_entity_id_6ea4b658 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_parent_audit_token_pid_to_process_parent_pid_2ae7dc56 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_add_instigator_parent_audit_token_euid_to_process_parent_user_id_e7e1d6cb field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_parent_audit_token_e_username_to_process_parent_user_name_440fee1a field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_add_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_d9b7529d field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_add_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_72ed6acb field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_4b044848 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_parent_audit_token_exec_path_to_process_parent_executable_ef4ef513 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_9032fb6e field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_responsible_audit_token_pid_to_process_responsible_pid_8ee9c6b8 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_add_instigator_responsible_audit_token_euid_to_process_responsible_user_id_ce20a155 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_responsible_audit_token_e_username_to_process_responsible_user_name_8bbb2422 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_add_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_159a2c19 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_add_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_d1d68a2f field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_d9865802 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_responsible_audit_token_exec_path_to_process_responsible_executable_cc7bbd81 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.responsible_audit_token.exec_path target_field: process.responsible.executable ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_attribute_value_remove.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_attribute_value_remove.yml index 1212995740a..53d0f602efa 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_attribute_value_remove.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_attribute_value_remove.yml @@ -2,132 +2,161 @@ description: Pipeline for parsing specific fields related to attribute value remove events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_115b7f3f field: event.reason value: Attribute removed from a user or group using Open Directory - append: + tag: append_event_type_505e05d4 field: event.type value: deletion - append: + tag: append_event_category_1c667e2c field: event.category - value: configuration + value: configuration - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_group_name_to_group_name_61de0f99 field: jamf_protect.telemetry.event.od_attribute_value_remove.group_name target_field: group.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_record_name_to_user_name_09bb579b field: jamf_protect.telemetry.event.od_attribute_value_remove.record_name target_field: user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_db_path_to_file_path_4551325f field: jamf_protect.telemetry.event.od_attribute_value_remove.db_path target_field: file.path ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_remove_attribute_name_to_jamf_protect_telemetry_attribute_name_d6f821db field: jamf_protect.telemetry.event.od_attribute_value_remove.attribute_name target_field: jamf_protect.telemetry.attribute_name type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_remove_attribute_value_to_jamf_protect_telemetry_attribute_value_45e4cd6f field: jamf_protect.telemetry.event.od_attribute_value_remove.attribute_value target_field: jamf_protect.telemetry.attribute_value type: string ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_start_time_to_process_start_66a72c19 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.start_time target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_audit_token_egid_to_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_audit_token_egid_63710a7a field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_audit_token_euid_to_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_audit_token_euid_a46f83b2 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.audit_token.euid type: string ignore_missing: true - append: + tag: append_user_effective_id_be38611f field: user.effective.id value: '{{{jamf_protect.telemetry.event.od_attribute_value_remove.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.od_attribute_value_remove?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_audit_token_e_username_to_process_user_name_1d546de0 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_is_platform_binary_to_process_platform_binary_29760324 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_is_es_client_to_process_endpoint_security_client_9aa0fd56 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_cdhash_to_process_hash_cdhash_034ab769 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_executable_sha1_to_process_hash_sha1_98a68cc1 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_executable_sha256_to_process_hash_sha256_d56a2799 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_tty_path_to_jamf_protect_telemetry_tty_de51595a field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_6751c6d9 field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.od_attribute_value_remove?.instigator?.tty != null - set: + tag: set_process_interactive_2da09e64 field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.od_attribute_value_remove?.instigator?.tty == null - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_audit_token_pid_to_process_pid_381cf218 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_2dd133b7 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_audit_token_uuid_to_process_entity_id_db7810c0 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_executable_path_to_process_executable_66abd87a field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -139,102 +168,123 @@ processors: } } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_db_path_to_process_working_directory_5bef5721 field: jamf_protect.telemetry.event.od_attribute_value_remove.db_path target_field: process.working_directory ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.od_attribute_value_remove?.db_path != null - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_signing_id_to_process_code_signature_signing_id_901860bd field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_team_id_to_process_code_signature_team_id_32c99937 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_codesigning_flags_to_process_code_signature_flags_c50a5e4a field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_parent_audit_token_uuid_to_process_parent_entity_id_d7f1a62d field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_parent_audit_token_pid_to_process_parent_pid_40b8e4ad field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_parent_audit_token_euid_to_process_parent_user_id_d3b84f44 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_parent_audit_token_e_username_to_process_parent_user_name_74352e3f field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_3966f6d0 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_26403068 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_add_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_4b044848 field: jamf_protect.telemetry.event.od_attribute_value_add.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_parent_audit_token_exec_path_to_process_parent_executable_903e4746 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_2fb759c7 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_responsible_audit_token_pid_to_process_responsible_pid_bd75c13b field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_responsible_audit_token_euid_to_process_responsible_user_id_1e6ed922 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_responsible_audit_token_e_username_to_process_responsible_user_name_b4726d37 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_80c34d30 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_abad566c field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_5885aac3 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_attribute_value_remove_instigator_responsible_audit_token_exec_path_to_process_responsible_executable_bc55f3c8 field: jamf_protect.telemetry.event.od_attribute_value_remove.instigator.responsible_audit_token.exec_path target_field: process.responsible.executable - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_create_group.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_create_group.yml index 0b837d040f4..db7689aa76e 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_create_group.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_create_group.yml @@ -2,123 +2,150 @@ description: Pipeline for parsing specific fields related to opendirectory group events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_370e5089 field: event.reason value: A group has been created using Open Directory - append: + tag: append_event_type_fbbdb681 field: event.type value: creation - append: + tag: append_event_category_1c667e2c field: event.category value: configuration - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_group_name_to_group_name_7bfdd6c2 field: jamf_protect.telemetry.event.od_create_group.group_name target_field: group.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_members_member_array_to_user_name_c322324a field: jamf_protect.telemetry.event.od_create_group.members.member_array target_field: user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_db_path_to_file_path_3bf66820 field: jamf_protect.telemetry.event.od_create_group.db_path target_field: file.path ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_start_time_to_process_start_a8567b62 field: jamf_protect.telemetry.event.od_create_group.instigator.start_time target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_create_group_instigator_audit_token_egid_to_jamf_protect_telemetry_event_od_create_group_instigator_audit_token_egid_cd3c4eee field: jamf_protect.telemetry.event.od_create_group.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.od_create_group.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_create_group_instigator_audit_token_euid_to_jamf_protect_telemetry_event_od_create_group_instigator_audit_token_euid_5265a43e field: jamf_protect.telemetry.event.od_create_group.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.od_create_group.instigator.audit_token.euid type: string ignore_missing: true - append: + tag: append_user_effective_id_5bdb3551 field: user.effective.id value: '{{{jamf_protect.telemetry.event.od_create_group.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.od_create_group?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_audit_token_e_username_to_process_user_name_722b03eb field: jamf_protect.telemetry.event.od_create_group.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_is_platform_binary_to_process_platform_binary_0944b0eb field: jamf_protect.telemetry.event.od_create_group.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_is_es_client_to_process_endpoint_security_client_a6b7deb7 field: jamf_protect.telemetry.event.od_create_group.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_cdhash_to_process_hash_cdhash_58bcc166 field: jamf_protect.telemetry.event.od_create_group.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_executable_sha1_to_process_hash_sha1_6b605a3c field: jamf_protect.telemetry.event.od_create_group.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_executable_sha256_to_process_hash_sha256_4bcca3a4 field: jamf_protect.telemetry.event.od_create_group.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_tty_path_to_jamf_protect_telemetry_tty_1efa2d9b field: jamf_protect.telemetry.event.od_create_group.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_7758f150 field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.od_create_group?.instigator?.tty != null - set: + tag: set_process_interactive_c266240b field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.od_create_group?.instigator?.tty == null - convert: + tag: convert_jamf_protect_telemetry_event_od_create_group_instigator_audit_token_pid_to_process_pid_f76811cf field: jamf_protect.telemetry.event.od_create_group.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_e3aa2840 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_audit_token_uuid_to_process_entity_id_33c53e87 field: jamf_protect.telemetry.event.od_create_group.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_executable_path_to_process_executable_0f972125 field: jamf_protect.telemetry.event.od_create_group.instigator.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -130,102 +157,123 @@ processors: } } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_db_path_to_process_working_directory_01028ecf field: jamf_protect.telemetry.event.od_create_group.db_path target_field: process.working_directory ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.od_create_group?.db_path != null - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_signing_id_to_process_code_signature_signing_id_a35d0836 field: jamf_protect.telemetry.event.od_create_group.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_team_id_to_process_code_signature_team_id_ebfa0d14 field: jamf_protect.telemetry.event.od_create_group.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_codesigning_flags_to_process_code_signature_flags_22715221 field: jamf_protect.telemetry.event.od_create_group.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_parent_audit_token_uuid_to_process_parent_entity_id_2de9eb6a field: jamf_protect.telemetry.event.od_create_group.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_parent_audit_token_pid_to_process_parent_pid_af1ef094 field: jamf_protect.telemetry.event.od_create_group.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_create_group_instigator_parent_audit_token_euid_to_process_parent_user_id_ca25736b field: jamf_protect.telemetry.event.od_create_group.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_parent_audit_token_e_username_to_process_parent_user_name_1c55db0c field: jamf_protect.telemetry.event.od_create_group.instigator.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_create_group_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_0814b4fd field: jamf_protect.telemetry.event.od_create_group.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_create_group_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_1530d5ab field: jamf_protect.telemetry.event.od_create_group.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_9b77032a field: jamf_protect.telemetry.event.od_create_group.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_parent_audit_token_exec_path_to_process_parent_executable_245424f5 field: jamf_protect.telemetry.event.od_create_group.instigator.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_0edddc20 field: jamf_protect.telemetry.event.od_create_group.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_responsible_audit_token_pid_to_process_responsible_pid_1d4bbebe field: jamf_protect.telemetry.event.od_create_group.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_create_group_instigator_responsible_audit_token_euid_to_process_responsible_user_id_baf88075 field: jamf_protect.telemetry.event.od_create_group.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_responsible_audit_token_e_username_to_process_responsible_user_name_7fd9045c field: jamf_protect.telemetry.event.od_create_group.instigator.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_create_group_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_1bc2acf9 field: jamf_protect.telemetry.event.od_create_group.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_create_group_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_578bb28f field: jamf_protect.telemetry.event.od_create_group.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_b288c77c field: jamf_protect.telemetry.event.od_create_group.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_group_instigator_responsible_audit_token_exec_path_to_process_responsible_executable_e113469b field: jamf_protect.telemetry.event.od_create_group.instigator.responsible_audit_token.exec_path target_field: process.responsible.executable - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_create_user.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_create_user.yml index 03b4b2ed80a..c93c8afd770 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_create_user.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_create_user.yml @@ -2,123 +2,150 @@ description: Pipeline for parsing specific fields related to opendirectory user events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_d250188f field: event.reason value: A user has been created using Open Directory - append: + tag: append_event_type_fbbdb681 field: event.type value: creation - append: + tag: append_event_category_1c667e2c field: event.category - value: configuration + value: configuration - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_group_name_to_group_name_4a2be660 field: jamf_protect.telemetry.event.od_create_user.group_name target_field: group.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_user_name_to_user_name_c5980de0 field: jamf_protect.telemetry.event.od_create_user.user_name target_field: user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_db_path_to_file_path_2b44be7e field: jamf_protect.telemetry.event.od_create_user.db_path target_field: file.path ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_start_time_to_process_start_81e6a02c field: jamf_protect.telemetry.event.od_create_user.instigator.start_time target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_create_user_instigator_audit_token_egid_to_jamf_protect_telemetry_event_od_create_user_instigator_audit_token_egid_c9a3254e field: jamf_protect.telemetry.event.od_create_user.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.od_create_user.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_create_user_instigator_audit_token_euid_to_jamf_protect_telemetry_event_od_create_user_instigator_audit_token_euid_89abd57e field: jamf_protect.telemetry.event.od_create_user.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.od_create_user.instigator.audit_token.euid type: string ignore_missing: true - append: + tag: append_user_effective_id_e011d519 field: user.effective.id value: '{{{jamf_protect.telemetry.event.od_create_user.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.od_create_user?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_audit_token_e_username_to_process_user_name_3c8863ad field: jamf_protect.telemetry.event.od_create_user.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_is_platform_binary_to_process_platform_binary_31964b71 field: jamf_protect.telemetry.event.od_create_user.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_is_es_client_to_process_endpoint_security_client_639a8dd1 field: jamf_protect.telemetry.event.od_create_user.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_cdhash_to_process_hash_cdhash_6ec24e54 field: jamf_protect.telemetry.event.od_create_user.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_executable_sha1_to_process_hash_sha1_28b9528a field: jamf_protect.telemetry.event.od_create_user.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_executable_sha256_to_process_hash_sha256_dd9dfe62 field: jamf_protect.telemetry.event.od_create_user.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_tty_path_to_jamf_protect_telemetry_tty_d26ea42d field: jamf_protect.telemetry.event.od_create_user.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_8ee39030 field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.od_create_user?.instigator?.tty != null - set: + tag: set_process_interactive_6a8a076b field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.od_create_user?.instigator?.tty == null - convert: + tag: convert_jamf_protect_telemetry_event_od_create_user_instigator_audit_token_pid_to_process_pid_e4d2fa7f field: jamf_protect.telemetry.event.od_create_user.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_f1b1c242 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_audit_token_uuid_to_process_entity_id_b6ef57a9 field: jamf_protect.telemetry.event.od_create_user.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_executable_path_to_process_executable_4d9b0097 field: jamf_protect.telemetry.event.od_create_user.instigator.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -130,102 +157,123 @@ processors: } } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_db_path_to_process_working_directory_bd1b828f field: jamf_protect.telemetry.event.od_create_user.db_path target_field: process.working_directory ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.od_create_user?.db_path != null - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_signing_id_to_process_code_signature_signing_id_d18a8edc field: jamf_protect.telemetry.event.od_create_user.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_team_id_to_process_code_signature_team_id_622e9342 field: jamf_protect.telemetry.event.od_create_user.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_codesigning_flags_to_process_code_signature_flags_aa767ac7 field: jamf_protect.telemetry.event.od_create_user.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_parent_audit_token_uuid_to_process_parent_entity_id_88a6d4cc field: jamf_protect.telemetry.event.od_create_user.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_parent_audit_token_pid_to_process_parent_pid_fcf85fda field: jamf_protect.telemetry.event.od_create_user.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_create_user_instigator_parent_audit_token_euid_to_process_parent_user_id_8851683b field: jamf_protect.telemetry.event.od_create_user.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_parent_audit_token_e_username_to_process_parent_user_name_6571cd8e field: jamf_protect.telemetry.event.od_create_user.instigator.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_create_user_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_dd8a2d4d field: jamf_protect.telemetry.event.od_create_user.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_create_user_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_04aef47b field: jamf_protect.telemetry.event.od_create_user.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_cb641f54 field: jamf_protect.telemetry.event.od_create_user.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_parent_audit_token_exec_path_to_process_parent_executable_035607df field: jamf_protect.telemetry.event.od_create_user.instigator.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_e3bae982 field: jamf_protect.telemetry.event.od_create_user.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_responsible_audit_token_pid_to_process_responsible_pid_030d494c field: jamf_protect.telemetry.event.od_create_user.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_create_user_instigator_responsible_audit_token_euid_to_process_responsible_user_id_fdc38905 field: jamf_protect.telemetry.event.od_create_user.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_create_user_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_05af1109 field: jamf_protect.telemetry.event.od_create_user.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_responsible_audit_token_e_username_to_process_responsible_user_name_b877baa6 field: jamf_protect.telemetry.event.od_create_user.instigator.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_create_user_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_1bba36df field: jamf_protect.telemetry.event.od_create_user.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_34e4a0de field: jamf_protect.telemetry.event.od_create_user.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_create_user_instigator_responsible_audit_token_exec_path_to_process_responsible_executable_f83c72cd field: jamf_protect.telemetry.event.od_create_user.instigator.responsible_audit_token.exec_path target_field: process.responsible.executable - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_delete_group.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_delete_group.yml index 5f54741ebd8..b56d5149308 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_delete_group.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_delete_group.yml @@ -2,123 +2,150 @@ description: Pipeline for parsing specific fields related to opendirectory group delete events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_bc99b82c field: event.reason value: A group has been deleted using Open Directory - append: + tag: append_event_type_505e05d4 field: event.type value: deletion - append: + tag: append_event_category_1c667e2c field: event.category - value: configuration + value: configuration - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_group_name_to_group_name_4afdd975 field: jamf_protect.telemetry.event.od_delete_group.group_name target_field: group.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_members_member_array_to_user_name_32640567 field: jamf_protect.telemetry.event.od_delete_group.members.member_array target_field: user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_db_path_to_file_path_0513af1b field: jamf_protect.telemetry.event.od_delete_group.db_path target_field: file.path ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_start_time_to_process_start_1f23656d field: jamf_protect.telemetry.event.od_delete_group.instigator.start_time target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_delete_group_instigator_audit_token_egid_to_jamf_protect_telemetry_event_od_delete_group_instigator_audit_token_egid_13306b66 field: jamf_protect.telemetry.event.od_delete_group.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.od_delete_group.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_delete_group_instigator_audit_token_euid_to_jamf_protect_telemetry_event_od_delete_group_instigator_audit_token_euid_5e8e094e field: jamf_protect.telemetry.event.od_delete_group.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.od_delete_group.instigator.audit_token.euid type: string ignore_missing: true - append: + tag: append_user_effective_id_db57d553 field: user.effective.id value: '{{{jamf_protect.telemetry.event.od_delete_group.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.od_delete_group?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_audit_token_e_username_to_process_user_name_51938274 field: jamf_protect.telemetry.event.od_delete_group.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_is_platform_binary_to_process_platform_binary_0106e490 field: jamf_protect.telemetry.event.od_delete_group.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_is_es_client_to_process_endpoint_security_client_db387812 field: jamf_protect.telemetry.event.od_delete_group.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_cdhash_to_process_hash_cdhash_5d364d15 field: jamf_protect.telemetry.event.od_delete_group.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_executable_sha1_to_process_hash_sha1_f7fe51cd field: jamf_protect.telemetry.event.od_delete_group.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_executable_sha256_to_process_hash_sha256_09143165 field: jamf_protect.telemetry.event.od_delete_group.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_tty_path_to_jamf_protect_telemetry_tty_55481c4e field: jamf_protect.telemetry.event.od_delete_group.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_ef804dc1 field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.od_delete_group?.instigator?.tty != null - set: + tag: set_process_interactive_788cca9c field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.od_delete_group?.instigator?.tty == null - convert: + tag: convert_jamf_protect_telemetry_event_od_delete_group_instigator_audit_token_pid_to_process_pid_21aae070 field: jamf_protect.telemetry.event.od_delete_group.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_4d82b623 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_audit_token_uuid_to_process_entity_id_9878b8a4 field: jamf_protect.telemetry.event.od_delete_group.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_executable_path_to_process_executable_1a19cb6e field: jamf_protect.telemetry.event.od_delete_group.instigator.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -130,102 +157,123 @@ processors: } } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_db_path_to_process_working_directory_225df6fd field: jamf_protect.telemetry.event.od_delete_group.db_path target_field: process.working_directory ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.od_delete_group?.db_path != null - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_signing_id_to_process_code_signature_signing_id_438a4391 field: jamf_protect.telemetry.event.od_delete_group.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_team_id_to_process_code_signature_team_id_ecc208db field: jamf_protect.telemetry.event.od_delete_group.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_codesigning_flags_to_process_code_signature_flags_53ffaefe field: jamf_protect.telemetry.event.od_delete_group.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_parent_audit_token_uuid_to_process_parent_entity_id_199da5e1 field: jamf_protect.telemetry.event.od_delete_group.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_parent_audit_token_pid_to_process_parent_pid_279c8d11 field: jamf_protect.telemetry.event.od_delete_group.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_delete_group_instigator_parent_audit_token_euid_to_process_parent_user_id_1e2827ec field: jamf_protect.telemetry.event.od_delete_group.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_parent_audit_token_e_username_to_process_parent_user_name_c4444373 field: jamf_protect.telemetry.event.od_delete_group.instigator.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_delete_group_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_2491d208 field: jamf_protect.telemetry.event.od_delete_group.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_delete_group_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_7da3f080 field: jamf_protect.telemetry.event.od_delete_group.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_4659eb71 field: jamf_protect.telemetry.event.od_delete_group.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_parent_audit_token_exec_path_to_process_parent_executable_87c883d2 field: jamf_protect.telemetry.event.od_delete_group.instigator.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_a5dca77b field: jamf_protect.telemetry.event.od_delete_group.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_responsible_audit_token_pid_to_process_responsible_pid_9136038f field: jamf_protect.telemetry.event.od_delete_group.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_delete_group_instigator_responsible_audit_token_euid_to_process_responsible_user_id_edede4aa field: jamf_protect.telemetry.event.od_delete_group.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_responsible_audit_token_e_username_to_process_responsible_user_name_de1cf99b field: jamf_protect.telemetry.event.od_delete_group.instigator.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_delete_group_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_6fd144c8 field: jamf_protect.telemetry.event.od_delete_group.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_delete_group_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_2dbbe5c4 field: jamf_protect.telemetry.event.od_delete_group.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_bd6de55f field: jamf_protect.telemetry.event.od_delete_group.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_group_instigator_responsible_audit_token_exec_path_to_process_responsible_executable_1fd85244 field: jamf_protect.telemetry.event.od_delete_group.instigator.responsible_audit_token.exec_path target_field: process.responsible.executable - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_delete_user.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_delete_user.yml index 397daa88dfe..d0c47313763 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_delete_user.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_delete_user.yml @@ -2,123 +2,150 @@ description: Pipeline for parsing specific fields related to opendirectory user delete events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_c81cf212 field: event.reason value: A user has been deleted using Open Directory - append: + tag: append_event_type_fbbdb681 field: event.type value: creation - append: + tag: append_event_category_1c667e2c field: event.category - value: configuration + value: configuration - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_group_name_to_group_name_73d9dde5 field: jamf_protect.telemetry.event.od_delete_user.group_name target_field: group.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_user_name_to_user_name_f000e065 field: jamf_protect.telemetry.event.od_delete_user.user_name target_field: user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_db_path_to_file_path_c101df8b field: jamf_protect.telemetry.event.od_delete_user.db_path target_field: file.path ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_start_time_to_process_start_ea03b67d field: jamf_protect.telemetry.event.od_delete_user.instigator.start_time target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_delete_user_instigator_audit_token_egid_to_jamf_protect_telemetry_event_od_delete_user_instigator_audit_token_egid_d9cbe6c0 field: jamf_protect.telemetry.event.od_delete_user.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.od_delete_user.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_delete_user_instigator_audit_token_euid_to_jamf_protect_telemetry_event_od_delete_user_instigator_audit_token_euid_3bf3d518 field: jamf_protect.telemetry.event.od_delete_user.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.od_delete_user.instigator.audit_token.euid type: string ignore_missing: true - append: + tag: append_user_effective_id_88305e85 field: user.effective.id value: '{{{jamf_protect.telemetry.event.od_delete_user.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.od_delete_user?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_audit_token_e_username_to_process_user_name_f9ede2e4 field: jamf_protect.telemetry.event.od_delete_user.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_is_platform_binary_to_process_platform_binary_bf5fd7a0 field: jamf_protect.telemetry.event.od_delete_user.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_is_es_client_to_process_endpoint_security_client_c2927ca2 field: jamf_protect.telemetry.event.od_delete_user.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_cdhash_to_process_hash_cdhash_be19a6c5 field: jamf_protect.telemetry.event.od_delete_user.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_executable_sha1_to_process_hash_sha1_ea6a6cbd field: jamf_protect.telemetry.event.od_delete_user.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_executable_sha256_to_process_hash_sha256_3fc0a395 field: jamf_protect.telemetry.event.od_delete_user.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_tty_path_to_jamf_protect_telemetry_tty_05d6c55e field: jamf_protect.telemetry.event.od_delete_user.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_e8a62f2b field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.od_delete_user?.instigator?.tty != null - set: + tag: set_process_interactive_1475b8ae field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.od_delete_user?.instigator?.tty == null - convert: + tag: convert_jamf_protect_telemetry_event_od_delete_user_instigator_audit_token_pid_to_process_pid_96e33f92 field: jamf_protect.telemetry.event.od_delete_user.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_1a8b79e3 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_audit_token_uuid_to_process_entity_id_fb5b9234 field: jamf_protect.telemetry.event.od_delete_user.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_executable_path_to_process_executable_262d1dbe field: jamf_protect.telemetry.event.od_delete_user.instigator.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -130,102 +157,123 @@ processors: } } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_db_path_to_process_working_directory_e55fd2d3 field: jamf_protect.telemetry.event.od_delete_user.db_path target_field: process.working_directory ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.od_delete_user?.db_path != null - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_signing_id_to_process_code_signature_signing_id_3792afc1 field: jamf_protect.telemetry.event.od_delete_user.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_team_id_to_process_code_signature_team_id_603563cb field: jamf_protect.telemetry.event.od_delete_user.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_codesigning_flags_to_process_code_signature_flags_4921f8ee field: jamf_protect.telemetry.event.od_delete_user.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_parent_audit_token_uuid_to_process_parent_entity_id_500d8a51 field: jamf_protect.telemetry.event.od_delete_user.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_parent_audit_token_pid_to_process_parent_pid_b74bd921 field: jamf_protect.telemetry.event.od_delete_user.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_delete_user_instigator_parent_audit_token_euid_to_process_parent_user_id_14968e7e field: jamf_protect.telemetry.event.od_delete_user.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_parent_audit_token_e_username_to_process_parent_user_name_0fad8903 field: jamf_protect.telemetry.event.od_delete_user.instigator.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_delete_user_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_66760702 field: jamf_protect.telemetry.event.od_delete_user.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_delete_user_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_3eb3d63e field: jamf_protect.telemetry.event.od_delete_user.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_9b659681 field: jamf_protect.telemetry.event.od_delete_user.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_parent_audit_token_exec_path_to_process_parent_executable_d5cce9a2 field: jamf_protect.telemetry.event.od_delete_user.instigator.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_8c8e26eb field: jamf_protect.telemetry.event.od_delete_user.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_responsible_audit_token_pid_to_process_responsible_pid_bbade39f field: jamf_protect.telemetry.event.od_delete_user.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_delete_user_instigator_responsible_audit_token_euid_to_process_responsible_user_id_9f35577c field: jamf_protect.telemetry.event.od_delete_user.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_responsible_audit_token_e_username_to_process_responsible_user_name_54cb626b field: jamf_protect.telemetry.event.od_delete_user.instigator.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_delete_user_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_cbddaa9a field: jamf_protect.telemetry.event.od_delete_user.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_delete_user_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_080f7ea2 field: jamf_protect.telemetry.event.od_delete_user.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_ef8e2aef field: jamf_protect.telemetry.event.od_delete_user.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_delete_user_instigator_responsible_audit_token_exec_path_to_process_responsible_executable_613087d4 field: jamf_protect.telemetry.event.od_delete_user.instigator.responsible_audit_token.exec_path target_field: process.responsible.executable - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_disable_user.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_disable_user.yml index c31e79cd872..331f86f89fc 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_disable_user.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_disable_user.yml @@ -2,123 +2,150 @@ description: Pipeline for parsing specific fields related to opendirectory disable user events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_27d94123 field: event.reason value: A user has been disabled using Open Directory - append: + tag: append_event_type_0602e5be field: event.type value: change - append: + tag: append_event_category_1c667e2c field: event.category - value: configuration + value: configuration - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_group_name_to_group_name_39e7d254 field: jamf_protect.telemetry.event.od_disable_user.group_name target_field: group.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_user_name_to_user_name_d0054354 field: jamf_protect.telemetry.event.od_disable_user.user_name target_field: user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_db_path_to_file_path_ce63d762 field: jamf_protect.telemetry.event.od_disable_user.db_path target_field: file.path ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_start_time_to_process_start_a7aaffa8 field: jamf_protect.telemetry.event.od_disable_user.instigator.start_time target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_disable_user_instigator_audit_token_egid_to_jamf_protect_telemetry_event_od_disable_user_instigator_audit_token_egid_a3102766 field: jamf_protect.telemetry.event.od_disable_user.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.od_disable_user.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_disable_user_instigator_audit_token_euid_to_jamf_protect_telemetry_event_od_disable_user_instigator_audit_token_euid_762087be field: jamf_protect.telemetry.event.od_disable_user.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.od_disable_user.instigator.audit_token.euid type: string ignore_missing: true - append: + tag: append_user_effective_id_2cd26f2d field: user.effective.id value: '{{{jamf_protect.telemetry.event.od_disable_user.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.od_disable_user?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_audit_token_e_username_to_process_user_name_d7e29669 field: jamf_protect.telemetry.event.od_disable_user.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_is_platform_binary_to_process_platform_binary_b1fe5a25 field: jamf_protect.telemetry.event.od_disable_user.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_is_es_client_to_process_endpoint_security_client_c02f1af5 field: jamf_protect.telemetry.event.od_disable_user.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_cdhash_to_process_hash_cdhash_3e826368 field: jamf_protect.telemetry.event.od_disable_user.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_executable_sha1_to_process_hash_sha1_0957387e field: jamf_protect.telemetry.event.od_disable_user.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_executable_sha256_to_process_hash_sha256_32b80b36 field: jamf_protect.telemetry.event.od_disable_user.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_tty_path_to_jamf_protect_telemetry_tty_165e2c39 field: jamf_protect.telemetry.event.od_disable_user.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_b9c0295a field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.od_disable_user?.instigator?.tty != null - set: + tag: set_process_interactive_fd77afdd field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.od_disable_user?.instigator?.tty == null - convert: + tag: convert_jamf_protect_telemetry_event_od_disable_user_instigator_audit_token_pid_to_process_pid_a6fac375 field: jamf_protect.telemetry.event.od_disable_user.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_7f0f0522 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_audit_token_uuid_to_process_entity_id_e715dd05 field: jamf_protect.telemetry.event.od_disable_user.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_executable_path_to_process_executable_2b47a803 field: jamf_protect.telemetry.event.od_disable_user.instigator.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -130,102 +157,123 @@ processors: } } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_db_path_to_process_working_directory_789f1153 field: jamf_protect.telemetry.event.od_disable_user.db_path target_field: process.working_directory ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.od_disable_user?.db_path != null - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_signing_id_to_process_code_signature_signing_id_67671dd8 field: jamf_protect.telemetry.event.od_disable_user.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_team_id_to_process_code_signature_team_id_abbc74ee field: jamf_protect.telemetry.event.od_disable_user.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_codesigning_flags_to_process_code_signature_flags_15590763 field: jamf_protect.telemetry.event.od_disable_user.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_parent_audit_token_uuid_to_process_parent_entity_id_f94a1a08 field: jamf_protect.telemetry.event.od_disable_user.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_parent_audit_token_pid_to_process_parent_pid_a49850a6 field: jamf_protect.telemetry.event.od_disable_user.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_disable_user_instigator_parent_audit_token_euid_to_process_parent_user_id_446af069 field: jamf_protect.telemetry.event.od_disable_user.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_parent_audit_token_e_username_to_process_parent_user_name_222772ea field: jamf_protect.telemetry.event.od_disable_user.instigator.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_disable_user_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_6cf0d80b field: jamf_protect.telemetry.event.od_disable_user.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_disable_user_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_ec12d47d field: jamf_protect.telemetry.event.od_disable_user.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_4094c018 field: jamf_protect.telemetry.event.od_disable_user.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_parent_audit_token_exec_path_to_process_parent_executable_1a79d1e3 field: jamf_protect.telemetry.event.od_disable_user.instigator.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_ce0e345e field: jamf_protect.telemetry.event.od_disable_user.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_responsible_audit_token_pid_to_process_responsible_pid_93632088 field: jamf_protect.telemetry.event.od_disable_user.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_disable_user_instigator_responsible_audit_token_euid_to_process_responsible_user_id_a7fa43cb field: jamf_protect.telemetry.event.od_disable_user.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_responsible_audit_token_e_username_to_process_responsible_user_name_9e5e57b2 field: jamf_protect.telemetry.event.od_disable_user.instigator.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_disable_user_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_63711167 field: jamf_protect.telemetry.event.od_disable_user.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_disable_user_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_644fb731 field: jamf_protect.telemetry.event.od_disable_user.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_ab8094d2 field: jamf_protect.telemetry.event.od_disable_user.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_disable_user_instigator_responsible_audit_token_exec_path_to_process_responsible_executable_e1e9f011 field: jamf_protect.telemetry.event.od_disable_user.instigator.responsible_audit_token.exec_path target_field: process.responsible.executable - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_enable_user.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_enable_user.yml index 55305effb60..5c2cfde47df 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_enable_user.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_enable_user.yml @@ -2,123 +2,150 @@ description: Pipeline for parsing specific fields related to opendirectory enable user events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_177f8556 field: event.reason value: A user has been enabled using Open Directory - append: + tag: append_event_type_0602e5be field: event.type value: change - append: + tag: append_event_category_1c667e2c field: event.category - value: configuration + value: configuration - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_group_name_to_group_name_0ae83bc1 field: jamf_protect.telemetry.event.od_enable_user.group_name target_field: group.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_user_name_to_user_name_ac0ee5d1 field: jamf_protect.telemetry.event.od_enable_user.user_name target_field: user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_db_path_to_file_path_f14c6cc7 field: jamf_protect.telemetry.event.od_enable_user.db_path target_field: file.path ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_start_time_to_process_start_68504901 field: jamf_protect.telemetry.event.od_enable_user.instigator.start_time target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_enable_user_instigator_audit_token_egid_to_jamf_protect_telemetry_event_od_enable_user_instigator_audit_token_egid_592c3e30 field: jamf_protect.telemetry.event.od_enable_user.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.od_enable_user.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_enable_user_instigator_audit_token_euid_to_jamf_protect_telemetry_event_od_enable_user_instigator_audit_token_euid_0999afd8 field: jamf_protect.telemetry.event.od_enable_user.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.od_enable_user.instigator.audit_token.euid type: string ignore_missing: true - append: + tag: append_user_effective_id_b377f255 field: user.effective.id value: '{{{jamf_protect.telemetry.event.od_enable_user.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.od_enable_user?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_audit_token_e_username_to_process_user_name_a53fd298 field: jamf_protect.telemetry.event.od_enable_user.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_is_platform_binary_to_process_platform_binary_0064957c field: jamf_protect.telemetry.event.od_enable_user.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_is_es_client_to_process_endpoint_security_client_5d76798e field: jamf_protect.telemetry.event.od_enable_user.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_cdhash_to_process_hash_cdhash_7f1581d1 field: jamf_protect.telemetry.event.od_enable_user.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_executable_sha1_to_process_hash_sha1_0c3bf909 field: jamf_protect.telemetry.event.od_enable_user.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_executable_sha256_to_process_hash_sha256_1a4954e1 field: jamf_protect.telemetry.event.od_enable_user.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_tty_path_to_jamf_protect_telemetry_tty_603f8cf2 field: jamf_protect.telemetry.event.od_enable_user.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_1eedd307 field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.od_enable_user?.instigator?.tty != null - set: + tag: set_process_interactive_5572747a field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.od_enable_user?.instigator?.tty == null - convert: + tag: convert_jamf_protect_telemetry_event_od_enable_user_instigator_audit_token_pid_to_process_pid_346ed426 field: jamf_protect.telemetry.event.od_enable_user.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_8477a1ff field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_audit_token_uuid_to_process_entity_id_10eac018 field: jamf_protect.telemetry.event.od_enable_user.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_executable_path_to_process_executable_1a231592 field: jamf_protect.telemetry.event.od_enable_user.instigator.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -130,102 +157,123 @@ processors: } } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_db_path_to_process_working_directory_f3d1998b field: jamf_protect.telemetry.event.od_enable_user.db_path target_field: process.working_directory ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.od_enable_user?.db_path != null - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_signing_id_to_process_code_signature_signing_id_9aee76a5 field: jamf_protect.telemetry.event.od_enable_user.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_team_id_to_process_code_signature_team_id_dc2f701f field: jamf_protect.telemetry.event.od_enable_user.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_codesigning_flags_to_process_code_signature_flags_9a638072 field: jamf_protect.telemetry.event.od_enable_user.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_parent_audit_token_uuid_to_process_parent_entity_id_be439025 field: jamf_protect.telemetry.event.od_enable_user.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_parent_audit_token_pid_to_process_parent_pid_c1747395 field: jamf_protect.telemetry.event.od_enable_user.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_enable_user_instigator_parent_audit_token_euid_to_process_parent_user_id_9061b39a field: jamf_protect.telemetry.event.od_enable_user.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_parent_audit_token_e_username_to_process_parent_user_name_153bc057 field: jamf_protect.telemetry.event.od_enable_user.instigator.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_enable_user_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_90b35406 field: jamf_protect.telemetry.event.od_enable_user.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_enable_user_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_1d5b3482 field: jamf_protect.telemetry.event.od_enable_user.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_d6791ded field: jamf_protect.telemetry.event.od_enable_user.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_parent_audit_token_exec_path_to_process_parent_executable_91938aae field: jamf_protect.telemetry.event.od_enable_user.instigator.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_0b2cd81f field: jamf_protect.telemetry.event.od_enable_user.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_responsible_audit_token_pid_to_process_responsible_pid_e964e623 field: jamf_protect.telemetry.event.od_enable_user.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_enable_user_instigator_responsible_audit_token_euid_to_process_responsible_user_id_73f4b088 field: jamf_protect.telemetry.event.od_enable_user.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_responsible_audit_token_e_username_to_process_responsible_user_name_e6980fef field: jamf_protect.telemetry.event.od_enable_user.instigator.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_enable_user_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_ccb3eb2e field: jamf_protect.telemetry.event.od_enable_user.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_enable_user_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_0dcda9a6 field: jamf_protect.telemetry.event.od_enable_user.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_ec9494eb field: jamf_protect.telemetry.event.od_enable_user.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_enable_user_instigator_responsible_audit_token_exec_path_to_process_responsible_executable_de5789d0 field: jamf_protect.telemetry.event.od_enable_user.instigator.responsible_audit_token.exec_path target_field: process.responsible.executable - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_group_add.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_group_add.yml index d4d0d99e87d..eb5ffdd9880 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_group_add.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_group_add.yml @@ -2,126 +2,154 @@ description: Pipeline for parsing specific fields related to opendirectory group added events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_b6e10a65 field: event.reason value: A member has been added to a group using Open Directory - append: + tag: append_event_type_0602e5be field: event.type value: change - append: + tag: append_event_type_fbbdb681 field: event.type value: creation - append: + tag: append_event_category_1c667e2c field: event.category - value: configuration + value: configuration - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_group_name_to_group_name_d29516c9 field: jamf_protect.telemetry.event.od_group_add.group_name target_field: group.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_member_member_value_to_user_name_9560d992 field: jamf_protect.telemetry.event.od_group_add.member.member_value target_field: user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_db_path_to_file_path_6abe018f field: jamf_protect.telemetry.event.od_group_add.db_path target_field: file.path ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_start_time_to_process_start_e8de8ee9 field: jamf_protect.telemetry.event.od_group_add.instigator.start_time target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_add_instigator_audit_token_egid_to_jamf_protect_telemetry_event_od_group_add_instigator_audit_token_egid_aac12680 field: jamf_protect.telemetry.event.od_group_add.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.od_group_add.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_add_instigator_audit_token_euid_to_jamf_protect_telemetry_event_od_group_add_instigator_audit_token_euid_5cb4cc34 field: jamf_protect.telemetry.event.od_group_add.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.od_group_add.instigator.audit_token.euid type: string ignore_missing: true - append: + tag: append_user_effective_id_5afd34d1 field: user.effective.id value: '{{{jamf_protect.telemetry.event.od_group_add.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.od_group_add?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_audit_token_e_username_to_process_user_name_e14bc8d0 field: jamf_protect.telemetry.event.od_group_add.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_is_platform_binary_to_process_platform_binary_efad1074 field: jamf_protect.telemetry.event.od_group_add.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_is_es_client_to_process_endpoint_security_client_930f88e6 field: jamf_protect.telemetry.event.od_group_add.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_cdhash_to_process_hash_cdhash_f37d5559 field: jamf_protect.telemetry.event.od_group_add.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_executable_sha1_to_process_hash_sha1_5515b3f1 field: jamf_protect.telemetry.event.od_group_add.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_executable_sha256_to_process_hash_sha256_d095e209 field: jamf_protect.telemetry.event.od_group_add.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_tty_path_to_jamf_protect_telemetry_tty_5ba4e5aa field: jamf_protect.telemetry.event.od_group_add.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_b14fe477 field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.od_group_add?.instigator?.tty != null - set: + tag: set_process_interactive_e735bd2a field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.od_group_add?.instigator?.tty == null - convert: + tag: convert_jamf_protect_telemetry_event_od_group_add_instigator_audit_token_pid_to_process_pid_186ac112 field: jamf_protect.telemetry.event.od_group_add.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_0f818f1b field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_audit_token_uuid_to_process_entity_id_8651d710 field: jamf_protect.telemetry.event.od_group_add.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_executable_path_to_process_executable_400d88ca field: jamf_protect.telemetry.event.od_group_add.instigator.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -133,102 +161,123 @@ processors: } } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_db_path_to_process_working_directory_e54d8887 field: jamf_protect.telemetry.event.od_group_add.db_path target_field: process.working_directory ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.od_group_add?.db_path != null - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_signing_id_to_process_code_signature_signing_id_ecd8a6ed field: jamf_protect.telemetry.event.od_group_add.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_team_id_to_process_code_signature_team_id_2849a9a7 field: jamf_protect.telemetry.event.od_group_add.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_codesigning_flags_to_process_code_signature_flags_d33ad33a field: jamf_protect.telemetry.event.od_group_add.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_parent_audit_token_uuid_to_process_parent_entity_id_ad20c9dd field: jamf_protect.telemetry.event.od_group_add.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_parent_audit_token_pid_to_process_parent_pid_586801bd field: jamf_protect.telemetry.event.od_group_add.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_add_instigator_parent_audit_token_euid_to_process_parent_user_id_615423fe field: jamf_protect.telemetry.event.od_group_add.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_parent_audit_token_e_username_to_process_parent_user_name_71329e0f field: jamf_protect.telemetry.event.od_group_add.instigator.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_add_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_f5d56782 field: jamf_protect.telemetry.event.od_group_add.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_add_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_71d8bfbe field: jamf_protect.telemetry.event.od_group_add.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_d67298d5 field: jamf_protect.telemetry.event.od_group_add.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_parent_audit_token_exec_path_to_process_parent_executable_706c6516 field: jamf_protect.telemetry.event.od_group_add.instigator.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_b407cbf7 field: jamf_protect.telemetry.event.od_group_add.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_responsible_audit_token_pid_to_process_responsible_pid_88f3d5cb field: jamf_protect.telemetry.event.od_group_add.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_add_instigator_responsible_audit_token_euid_to_process_responsible_user_id_3c1f58fc field: jamf_protect.telemetry.event.od_group_add.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_responsible_audit_token_e_username_to_process_responsible_user_name_330b0207 field: jamf_protect.telemetry.event.od_group_add.instigator.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_add_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_33074f1a field: jamf_protect.telemetry.event.od_group_add.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_add_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_ee9d7422 field: jamf_protect.telemetry.event.od_group_add.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_447d8b53 field: jamf_protect.telemetry.event.od_group_add.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_add_instigator_responsible_audit_token_exec_path_to_process_responsible_executable_98d64a58 field: jamf_protect.telemetry.event.od_group_add.instigator.responsible_audit_token.exec_path target_field: process.responsible.executable - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_group_remove.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_group_remove.yml index aa24456669f..8d905290744 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_group_remove.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_group_remove.yml @@ -2,126 +2,154 @@ description: Pipeline for parsing specific fields related to opendirectory group removed events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_eac587be field: event.reason value: A member has been removed from a group using Open Directory - append: + tag: append_event_type_0602e5be field: event.type value: change - append: + tag: append_event_category_1c667e2c field: event.category value: configuration - append: + tag: append_event_type_505e05d4 field: event.type - value: deletion + value: deletion - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_group_name_to_group_name_8c88c72a field: jamf_protect.telemetry.event.od_group_remove.group_name target_field: group.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_member_member_value_to_user_name_93527c15 field: jamf_protect.telemetry.event.od_group_remove.member.member_value target_field: user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_db_path_to_file_path_cf60a768 field: jamf_protect.telemetry.event.od_group_remove.db_path target_field: file.path ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_start_time_to_process_start_bb641e4a field: jamf_protect.telemetry.event.od_group_remove.instigator.start_time target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_remove_instigator_audit_token_egid_to_jamf_protect_telemetry_event_od_group_remove_instigator_audit_token_egid_3a4408a2 field: jamf_protect.telemetry.event.od_group_remove.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.od_group_remove.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_remove_instigator_audit_token_euid_to_jamf_protect_telemetry_event_od_group_remove_instigator_audit_token_euid_e970ba0e field: jamf_protect.telemetry.event.od_group_remove.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.od_group_remove.instigator.audit_token.euid type: string ignore_missing: true - append: + tag: append_user_effective_id_07fcd4e9 field: user.effective.id value: '{{{jamf_protect.telemetry.event.od_group_remove.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.od_group_remove?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_audit_token_e_username_to_process_user_name_143967c3 field: jamf_protect.telemetry.event.od_group_remove.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_is_platform_binary_to_process_platform_binary_71d4c403 field: jamf_protect.telemetry.event.od_group_remove.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_is_es_client_to_process_endpoint_security_client_73af412f field: jamf_protect.telemetry.event.od_group_remove.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_cdhash_to_process_hash_cdhash_119ba8ee field: jamf_protect.telemetry.event.od_group_remove.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_executable_sha1_to_process_hash_sha1_f3e276c4 field: jamf_protect.telemetry.event.od_group_remove.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_executable_sha256_to_process_hash_sha256_cb89bb6c field: jamf_protect.telemetry.event.od_group_remove.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_tty_path_to_jamf_protect_telemetry_tty_e69f9333 field: jamf_protect.telemetry.event.od_group_remove.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_677cb2d0 field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.od_group_remove?.instigator?.tty != null - set: + tag: set_process_interactive_4aafc08b field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.od_group_remove?.instigator?.tty == null - convert: + tag: convert_jamf_protect_telemetry_event_od_group_remove_instigator_audit_token_pid_to_process_pid_be696353 field: jamf_protect.telemetry.event.od_group_remove.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_5fb29ce4 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_audit_token_uuid_to_process_entity_id_889b50bf field: jamf_protect.telemetry.event.od_group_remove.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_executable_path_to_process_executable_759fb4fd field: jamf_protect.telemetry.event.od_group_remove.instigator.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -133,102 +161,123 @@ processors: } } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_db_path_to_process_working_directory_41cbc02f field: jamf_protect.telemetry.event.od_group_remove.db_path target_field: process.working_directory ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.od_group_remove?.db_path != null - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_signing_id_to_process_code_signature_signing_id_9c32b85e field: jamf_protect.telemetry.event.od_group_remove.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_team_id_to_process_code_signature_team_id_6264b09c field: jamf_protect.telemetry.event.od_group_remove.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_codesigning_flags_to_process_code_signature_flags_6a776dc9 field: jamf_protect.telemetry.event.od_group_remove.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_parent_audit_token_uuid_to_process_parent_entity_id_49a70842 field: jamf_protect.telemetry.event.od_group_remove.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_parent_audit_token_pid_to_process_parent_pid_142a9adc field: jamf_protect.telemetry.event.od_group_remove.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_remove_instigator_parent_audit_token_euid_to_process_parent_user_id_8fa99397 field: jamf_protect.telemetry.event.od_group_remove.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_parent_audit_token_e_username_to_process_parent_user_name_a9630a04 field: jamf_protect.telemetry.event.od_group_remove.instigator.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_remove_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_a4d842e1 field: jamf_protect.telemetry.event.od_group_remove.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_remove_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_d091093f field: jamf_protect.telemetry.event.od_group_remove.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_f1009692 field: jamf_protect.telemetry.event.od_group_remove.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_parent_audit_token_exec_path_to_process_parent_executable_079e8bfd field: jamf_protect.telemetry.event.od_group_remove.instigator.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_f0b2a398 field: jamf_protect.telemetry.event.od_group_remove.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_responsible_audit_token_pid_to_process_responsible_pid_6062e606 field: jamf_protect.telemetry.event.od_group_remove.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_remove_instigator_responsible_audit_token_euid_to_process_responsible_user_id_f8e61af1 field: jamf_protect.telemetry.event.od_group_remove.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_responsible_audit_token_e_username_to_process_responsible_user_name_22637854 field: jamf_protect.telemetry.event.od_group_remove.instigator.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_remove_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_c80f5cad field: jamf_protect.telemetry.event.od_group_remove.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_remove_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_c18e5593 field: jamf_protect.telemetry.event.od_group_remove.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_0bd8da04 field: jamf_protect.telemetry.event.od_group_remove.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_remove_instigator_responsible_audit_token_exec_path_to_process_responsible_executable_a3c2c843 field: jamf_protect.telemetry.event.od_group_remove.instigator.responsible_audit_token.exec_path target_field: process.responsible.executable ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_group_set.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_group_set.yml index c2fef48b841..f4219d65c78 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_group_set.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_group_set.yml @@ -2,123 +2,150 @@ description: Pipeline for parsing specific fields related to group set events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_df6d1b9e field: event.reason value: A group has a member initialised or replaced using Open Directory - append: + tag: append_event_type_fbbdb681 field: event.type value: creation - append: + tag: append_event_category_1c667e2c field: event.category - value: configuration + value: configuration - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_group_name_to_group_name_07299b3e field: jamf_protect.telemetry.event.od_group_set.group_name target_field: group.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_members_member_array_to_user_name_84b8c6ce field: jamf_protect.telemetry.event.od_group_set.members.member_array target_field: user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_db_path_to_file_path_df3f9f1c field: jamf_protect.telemetry.event.od_group_set.db_path target_field: file.path ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_start_time_to_process_start_e9d39716 field: jamf_protect.telemetry.event.od_group_set.instigator.start_time target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_set_instigator_audit_token_egid_to_jamf_protect_telemetry_event_od_group_set_instigator_audit_token_egid_44158e9e field: jamf_protect.telemetry.event.od_group_set.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.od_group_set.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_set_instigator_audit_token_euid_to_jamf_protect_telemetry_event_od_group_set_instigator_audit_token_euid_8e21f7d2 field: jamf_protect.telemetry.event.od_group_set.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.od_group_set.instigator.audit_token.euid type: string ignore_missing: true - append: + tag: append_user_effective_id_04e9f309 field: user.effective.id value: '{{{jamf_protect.telemetry.event.od_group_set.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.od_group_set?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_audit_token_e_username_to_process_user_name_b6fcb3cf field: jamf_protect.telemetry.event.od_group_set.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_is_platform_binary_to_process_platform_binary_1407ba17 field: jamf_protect.telemetry.event.od_group_set.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_is_es_client_to_process_endpoint_security_client_5e3fd763 field: jamf_protect.telemetry.event.od_group_set.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_cdhash_to_process_hash_cdhash_3f8bc8e2 field: jamf_protect.telemetry.event.od_group_set.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_executable_sha1_to_process_hash_sha1_747fd448 field: jamf_protect.telemetry.event.od_group_set.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_executable_sha256_to_process_hash_sha256_97a4d970 field: jamf_protect.telemetry.event.od_group_set.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_tty_path_to_jamf_protect_telemetry_tty_0f49855f field: jamf_protect.telemetry.event.od_group_set.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_200e35b2 field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.od_group_set?.instigator?.tty != null - set: + tag: set_process_interactive_8acd25a5 field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.od_group_set?.instigator?.tty == null - convert: + tag: convert_jamf_protect_telemetry_event_od_group_set_instigator_audit_token_pid_to_process_pid_3c90f611 field: jamf_protect.telemetry.event.od_group_set.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_48df4fac field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_audit_token_uuid_to_process_entity_id_fa05583b field: jamf_protect.telemetry.event.od_group_set.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_executable_path_to_process_executable_362dbcc9 field: jamf_protect.telemetry.event.od_group_set.instigator.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -130,102 +157,123 @@ processors: } } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_db_path_to_process_working_directory_aecf48a7 field: jamf_protect.telemetry.event.od_group_set.db_path target_field: process.working_directory ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.od_group_set?.db_path != null - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_signing_id_to_process_code_signature_signing_id_b90e068a field: jamf_protect.telemetry.event.od_group_set.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_team_id_to_process_code_signature_team_id_d8ee0688 field: jamf_protect.telemetry.event.od_group_set.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_codesigning_flags_to_process_code_signature_flags_9692ac45 field: jamf_protect.telemetry.event.od_group_set.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_parent_audit_token_uuid_to_process_parent_entity_id_c32b430e field: jamf_protect.telemetry.event.od_group_set.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_parent_audit_token_pid_to_process_parent_pid_008fd978 field: jamf_protect.telemetry.event.od_group_set.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_set_instigator_parent_audit_token_euid_to_process_parent_user_id_2eeca7ad field: jamf_protect.telemetry.event.od_group_set.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_parent_audit_token_e_username_to_process_parent_user_name_aa486230 field: jamf_protect.telemetry.event.od_group_set.instigator.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_set_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_23016f47 field: jamf_protect.telemetry.event.od_group_set.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_set_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_35c91209 field: jamf_protect.telemetry.event.od_group_set.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_ec86f786 field: jamf_protect.telemetry.event.od_group_set.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_parent_audit_token_exec_path_to_process_parent_executable_192b2dd1 field: jamf_protect.telemetry.event.od_group_set.instigator.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_8a09f834 field: jamf_protect.telemetry.event.od_group_set.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_responsible_audit_token_pid_to_process_responsible_pid_fd9d7732 field: jamf_protect.telemetry.event.od_group_set.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_set_instigator_responsible_audit_token_euid_to_process_responsible_user_id_0d33fd7f field: jamf_protect.telemetry.event.od_group_set.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_responsible_audit_token_e_username_to_process_responsible_user_name_c5c4c280 field: jamf_protect.telemetry.event.od_group_set.instigator.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_set_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_a36a5ec3 field: jamf_protect.telemetry.event.od_group_set.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_group_set_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_ed8c0e1d field: jamf_protect.telemetry.event.od_group_set.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_7b6cd8c8 field: jamf_protect.telemetry.event.od_group_set.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_group_set_instigator_responsible_audit_token_exec_path_to_process_responsible_executable_a1a951c7 field: jamf_protect.telemetry.event.od_group_set.instigator.responsible_audit_token.exec_path target_field: process.responsible.executable - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_modify_password.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_modify_password.yml index f88e6176a9a..d6bddd66d3e 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_modify_password.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_od_modify_password.yml @@ -2,31 +2,38 @@ description: Pipeline for parsing specific fields related to password change events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_93c48557 field: event.reason value: A user password is modified via Open Directory - append: + tag: append_event_type_0602e5be field: event.type value: change - append: + tag: append_event_category_1c667e2c field: event.category - value: configuration + value: configuration - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_group_name_to_group_name_480c2e20 field: jamf_protect.telemetry.event.od_modify_password.group_name target_field: group.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_account_name_to_user_name_12c03950 field: jamf_protect.telemetry.event.od_modify_password.account_name target_field: user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_db_path_to_file_path_8e788c3e field: jamf_protect.telemetry.event.od_modify_password.db_path target_field: file.path ignore_missing: true - script: + tag: script_e7bfa755 lang: painless params: itemTypeMap: @@ -40,6 +47,7 @@ processors: ctx.jamf_protect.telemetry.account_type = itemTypeString; } - script: + tag: script_f0569cee lang: painless params: itemTypeMap: @@ -115,98 +123,119 @@ processors: ctx.jamf_protect.telemetry.error_message = itemTypeString; } -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_start_time_to_process_start_12a06c6c field: jamf_protect.telemetry.event.od_modify_password.instigator.start_time target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_modify_password_instigator_audit_token_egid_to_jamf_protect_telemetry_event_od_modify_password_instigator_audit_token_egid_2ce62952 field: jamf_protect.telemetry.event.od_modify_password.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.od_modify_password.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_modify_password_instigator_audit_token_euid_to_jamf_protect_telemetry_event_od_modify_password_instigator_audit_token_euid_ec34fcbe field: jamf_protect.telemetry.event.od_modify_password.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.od_modify_password.instigator.audit_token.euid type: string ignore_missing: true - append: + tag: append_user_effective_id_9c12f0bd field: user.effective.id value: '{{{jamf_protect.telemetry.event.od_modify_password.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.od_modify_password?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_audit_token_e_username_to_process_user_name_b04abaed field: jamf_protect.telemetry.event.od_modify_password.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_is_platform_binary_to_process_platform_binary_1f4cd9b1 field: jamf_protect.telemetry.event.od_modify_password.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_is_es_client_to_process_endpoint_security_client_7f869111 field: jamf_protect.telemetry.event.od_modify_password.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_cdhash_to_process_hash_cdhash_e959ba14 field: jamf_protect.telemetry.event.od_modify_password.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_executable_sha1_to_process_hash_sha1_1f4239ca field: jamf_protect.telemetry.event.od_modify_password.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_executable_sha256_to_process_hash_sha256_871d55a2 field: jamf_protect.telemetry.event.od_modify_password.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_tty_path_to_jamf_protect_telemetry_tty_31741d6d field: jamf_protect.telemetry.event.od_modify_password.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_4184fd74 field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.od_modify_password?.instigator?.tty != null - set: + tag: set_process_interactive_092e9f9f field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.od_modify_password?.instigator?.tty == null - convert: + tag: convert_jamf_protect_telemetry_event_od_modify_password_instigator_audit_token_pid_to_process_pid_0104c1a7 field: jamf_protect.telemetry.event.od_modify_password.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_8cb3436e field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_audit_token_uuid_to_process_entity_id_8b17a269 field: jamf_protect.telemetry.event.od_modify_password.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_executable_path_to_process_executable_25d12857 field: jamf_protect.telemetry.event.od_modify_password.instigator.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -218,103 +247,124 @@ processors: } } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_db_path_to_process_working_directory_ae8f65eb field: jamf_protect.telemetry.event.od_modify_password.db_path target_field: process.working_directory ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.od_modify_password?.db_path != null - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_signing_id_to_process_code_signature_signing_id_223bfa9c field: jamf_protect.telemetry.event.od_modify_password.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_team_id_to_process_code_signature_team_id_a6c62e02 field: jamf_protect.telemetry.event.od_modify_password.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_codesigning_flags_to_process_code_signature_flags_570f0b87 field: jamf_protect.telemetry.event.od_modify_password.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_parent_audit_token_uuid_to_process_parent_entity_id_3efc9a0c field: jamf_protect.telemetry.event.od_modify_password.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_parent_audit_token_pid_to_process_parent_pid_57cc891a field: jamf_protect.telemetry.event.od_modify_password.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_modify_password_instigator_parent_audit_token_euid_to_process_parent_user_id_e3315773 field: jamf_protect.telemetry.event.od_modify_password.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_parent_audit_token_e_username_to_process_parent_user_name_54190e4e field: jamf_protect.telemetry.event.od_modify_password.instigator.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_modify_password_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_556d2015 field: jamf_protect.telemetry.event.od_modify_password.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_modify_password_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_1edfe383 field: jamf_protect.telemetry.event.od_modify_password.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_6af8db14 field: jamf_protect.telemetry.event.od_modify_password.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_parent_audit_token_exec_path_to_process_parent_executable_1b1b3c1f field: jamf_protect.telemetry.event.od_modify_password.instigator.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_2a5d44c2 field: jamf_protect.telemetry.event.od_modify_password.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_responsible_audit_token_pid_to_process_responsible_pid_83c56b8c field: jamf_protect.telemetry.event.od_modify_password.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_modify_password_instigator_responsible_audit_token_euid_to_process_responsible_user_id_ba851b1d field: jamf_protect.telemetry.event.od_modify_password.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_responsible_audit_token_e_username_to_process_responsible_user_name_b62e7166 field: jamf_protect.telemetry.event.od_modify_password.instigator.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_modify_password_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_b02a1c31 field: jamf_protect.telemetry.event.od_modify_password.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_od_modify_password_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_987fc4c7 field: jamf_protect.telemetry.event.od_modify_password.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_6f86d19e field: jamf_protect.telemetry.event.od_modify_password.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_od_modify_password_instigator_responsible_audit_token_exec_path_to_process_responsible_executable_3fa1b10d field: jamf_protect.telemetry.event.od_modify_password.instigator.responsible_audit_token.exec_path target_field: process.responsible.executable - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_openssh_login.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_openssh_login.yml index a8c144b8726..b4331512167 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_openssh_login.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_openssh_login.yml @@ -2,22 +2,27 @@ description: Pipeline for parsing specific fields related to openssh login events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_e1452e22 field: event.reason value: A user has logged into the system via OpenSSH - append: + tag: append_event_type_fffb7a5a field: event.type value: start - append: + tag: append_event_category_109b4d72 field: event.category - value: authentication + value: authentication - append: + tag: append_event_category_d21db9a2 field: event.category - value: session + value: session - script: + tag: script_ce789f5f lang: painless source: | ctx.event = ctx.event != null ? ctx.event : new HashMap(); @@ -32,19 +37,23 @@ processors: ctx.event.outcome = 'unknown'; } - convert: + tag: convert_jamf_protect_telemetry_event_openssh_login_graphical_session_id_to_jamf_protect_telemetry_graphical_authentication_username_bbb900db field: jamf_protect.telemetry.event.openssh_login.graphical_session_id target_field: jamf_protect.telemetry.graphical_authentication_username type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_openssh_login_source_address_to_source_ip_8e1ab69f field: jamf_protect.telemetry.event.openssh_login.source_address target_field: source.ip ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_openssh_login_username_to_user_name_fad5dbb5 field: jamf_protect.telemetry.event.openssh_login.username target_field: user.name ignore_missing: true - script: + tag: script_8a6776b6 lang: painless params: itemTypeMap: @@ -60,6 +69,7 @@ processors: ctx.jamf_protect.telemetry.source_address_type = itemTypeString; } - script: + tag: script_4340498e lang: painless params: itemTypeMap: @@ -80,8 +90,9 @@ processors: ctx.jamf_protect = ctx.jamf_protect != null ? ctx.jamf_protect : new HashMap(); ctx.jamf_protect.telemetry.authentication_result_type = itemTypeString; } -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - pipeline: + tag: pipeline_0d75fa7a name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_openssh_logout.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_openssh_logout.yml index 5bac4027b32..738803c72fd 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_openssh_logout.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_openssh_logout.yml @@ -2,41 +2,50 @@ description: Pipeline for parsing specific fields related to openssh logout events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_d06cf2c4 field: event.reason value: A user has logged out of an OpenSSH session - append: + tag: append_event_type_fffb7a5a field: event.type value: start - append: + tag: append_event_category_109b4d72 field: event.category value: authentication - append: + tag: append_event_category_d21db9a2 field: event.category - value: session + value: session - convert: + tag: convert_jamf_protect_telemetry_event_openssh_logout_graphical_session_id_to_jamf_protect_telemetry_graphical_authentication_username_2604be48 field: jamf_protect.telemetry.event.openssh_logout.graphical_session_id target_field: jamf_protect.telemetry.graphical_authentication_username type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_openssh_logout_username_to_user_name_2e98da9e field: jamf_protect.telemetry.event.openssh_logout.username target_field: user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_openssh_logout_uid_to_user_id_5f81e66a field: jamf_protect.telemetry.event.openssh_logout.uid target_field: user.id type: string ignore_missing: true - + - rename: + tag: rename_jamf_protect_telemetry_event_openssh_logout_source_address_to_source_ip_8f4d330c field: jamf_protect.telemetry.event.openssh_logout.source_address target_field: source.ip ignore_missing: true - script: + tag: script_464af674 lang: painless params: itemTypeMap: @@ -52,8 +61,10 @@ processors: ctx.jamf_protect.telemetry.source_address_type = itemTypeString; } -########################## -## ECS Process ## -########################## + + ########################## + ## ECS Process ## + ########################## - pipeline: + tag: pipeline_0d75fa7a name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_profile_add.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_profile_add.yml index ff306b9e9be..375678c26b4 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_profile_add.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_profile_add.yml @@ -2,39 +2,47 @@ description: Pipeline for parsing specific fields related to profile added events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_7186cf49 field: event.reason value: A configuration profile is installed on the system - append: + tag: append_event_type_fbbdb681 field: event.type value: creation - append: + tag: append_event_category_1c667e2c field: event.category - value: configuration + value: configuration - convert: + tag: convert_jamf_protect_telemetry_event_profile_add_profile_scope_to_jamf_protect_telemetry_profile_scope_c4d1e50a field: jamf_protect.telemetry.event.profile_add.profile.scope target_field: jamf_protect.telemetry.profile_scope type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_add_profile_identifier_to_jamf_protect_telemetry_profile_identifier_0f462820 field: jamf_protect.telemetry.event.profile_add.profile.identifier target_field: jamf_protect.telemetry.profile_identifier type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_add_profile_uuid_to_jamf_protect_telemetry_profile_uuid_45c07284 field: jamf_protect.telemetry.event.profile_add.profile.uuid target_field: jamf_protect.telemetry.profile_uuid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_add_profile_display_name_to_jamf_protect_telemetry_profile_display_name_7caf4706 field: jamf_protect.telemetry.event.profile_add.profile.display_name target_field: jamf_protect.telemetry.profile_display_name type: string ignore_missing: true - script: + tag: script_9218c097 lang: painless params: itemTypeMap: @@ -48,108 +56,131 @@ processors: ctx.jamf_protect.telemetry.profile_install_source = itemTypeString; } - convert: + tag: convert_jamf_protect_telemetry_event_profile_add_profile_organization_to_jamf_protect_telemetry_profile_organization_edf3c4f8 field: jamf_protect.telemetry.event.profile_add.profile.organization target_field: jamf_protect.telemetry.profile_organization type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_add_is_update_to_jamf_protect_telemetry_profile_is_updated_c87444f5 field: jamf_protect.telemetry.event.profile_add.is_update target_field: jamf_protect.telemetry.profile_is_updated type: boolean ignore_missing: true ignore_failure: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_start_time_to_process_start_c60159e3 field: jamf_protect.telemetry.event.profile_add.instigator.start_time target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_add_instigator_audit_token_egid_to_jamf_protect_telemetry_event_profile_add_instigator_audit_token_egid_4b73be06 field: jamf_protect.telemetry.event.profile_add.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.profile_add.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_add_instigator_audit_token_euid_to_jamf_protect_telemetry_event_profile_add_instigator_audit_token_euid_50352d02 field: jamf_protect.telemetry.event.profile_add.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.profile_add.instigator.audit_token.euid type: string ignore_missing: true - append: + tag: append_user_effective_id_f4b846ab field: user.effective.id value: '{{{jamf_protect.telemetry.event.profile_add.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.profile_add?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_audit_token_e_username_to_process_user_name_18440f6a field: jamf_protect.telemetry.event.profile_add.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_is_platform_binary_to_process_platform_binary_994334da field: jamf_protect.telemetry.event.profile_add.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_is_es_client_to_process_endpoint_security_client_0fc5edc0 field: jamf_protect.telemetry.event.profile_add.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_cdhash_to_process_hash_cdhash_802ca9a7 field: jamf_protect.telemetry.event.profile_add.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_executable_sha1_to_process_hash_sha1_843962a7 field: jamf_protect.telemetry.event.profile_add.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_executable_sha256_to_process_hash_sha256_1c63cb3f field: jamf_protect.telemetry.event.profile_add.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_tty_path_to_jamf_protect_telemetry_tty_922eeb54 field: jamf_protect.telemetry.event.profile_add.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_2b95c25f field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.profile_add?.instigator?.tty != null - set: + tag: set_process_interactive_75ca09c2 field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.profile_add?.instigator?.tty == null - convert: + tag: convert_jamf_protect_telemetry_event_profile_add_instigator_audit_token_pid_to_process_pid_076e2db6 field: jamf_protect.telemetry.event.profile_add.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_9a1108d1 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_audit_token_uuid_to_process_entity_id_9de8f03a field: jamf_protect.telemetry.event.profile_add.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_executable_path_to_process_executable_f9271844 field: jamf_protect.telemetry.event.profile_add.instigator.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -161,102 +192,123 @@ processors: } } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_db_path_to_process_working_directory_d7622b4d field: jamf_protect.telemetry.event.profile_add.db_path target_field: process.working_directory ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.profile_add?.db_path != null - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_signing_id_to_process_code_signature_signing_id_613e9063 field: jamf_protect.telemetry.event.profile_add.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_team_id_to_process_code_signature_team_id_13d9623d field: jamf_protect.telemetry.event.profile_add.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_codesigning_flags_to_process_code_signature_flags_ce9bd1d0 field: jamf_protect.telemetry.event.profile_add.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_parent_audit_token_uuid_to_process_parent_entity_id_f010f5ff field: jamf_protect.telemetry.event.profile_add.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_parent_audit_token_pid_to_process_parent_pid_708c777b field: jamf_protect.telemetry.event.profile_add.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_add_instigator_parent_audit_token_euid_to_process_parent_user_id_cb6e47ca field: jamf_protect.telemetry.event.profile_add.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_parent_audit_token_e_username_to_process_parent_user_name_fac69de1 field: jamf_protect.telemetry.event.profile_add.instigator.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_add_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_c06955f6 field: jamf_protect.telemetry.event.profile_add.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_add_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_33a5afb2 field: jamf_protect.telemetry.event.profile_add.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_1e40407f field: jamf_protect.telemetry.event.profile_add.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_parent_audit_token_exec_path_to_process_parent_executable_7c7d5fe0 field: jamf_protect.telemetry.event.profile_add.instigator.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_f52aea71 field: jamf_protect.telemetry.event.profile_add.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_responsible_audit_token_pid_to_process_responsible_pid_dfd37609 field: jamf_protect.telemetry.event.profile_add.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_add_instigator_responsible_audit_token_euid_to_process_responsible_user_id_200e93b8 field: jamf_protect.telemetry.event.profile_add.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_responsible_audit_token_e_username_to_process_responsible_user_name_5e0d3529 field: jamf_protect.telemetry.event.profile_add.instigator.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_add_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_2de6399e field: jamf_protect.telemetry.event.profile_add.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_add_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_51d2e6d6 field: jamf_protect.telemetry.event.profile_add.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_a20c53b5 field: jamf_protect.telemetry.event.profile_add.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_add_instigator_responsible_audit_token_exec_path_to_process_responsible_executable_029e3e4a field: jamf_protect.telemetry.event.profile_add.instigator.responsible_audit_token.exec_path target_field: process.responsible.executable - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_profile_remove.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_profile_remove.yml index 70e8a002c20..374aff61551 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_profile_remove.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_profile_remove.yml @@ -2,39 +2,47 @@ description: Pipeline for parsing specific fields related to profile removed events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_626ece9a field: event.reason value: A configuration profile is removed from the system - append: + tag: append_event_type_fbbdb681 field: event.type value: creation - append: + tag: append_event_category_1c667e2c field: event.category - value: configuration + value: configuration - convert: + tag: convert_jamf_protect_telemetry_event_profile_remove_profile_scope_to_jamf_protect_telemetry_profile_scope_2dce57a1 field: jamf_protect.telemetry.event.profile_remove.profile.scope target_field: jamf_protect.telemetry.profile_scope type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_remove_profile_identifier_to_jamf_protect_telemetry_profile_identifier_6a449abb field: jamf_protect.telemetry.event.profile_remove.profile.identifier target_field: jamf_protect.telemetry.profile_identifier type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_remove_profile_uuid_to_jamf_protect_telemetry_profile_uuid_b509e61b field: jamf_protect.telemetry.event.profile_remove.profile.uuid target_field: jamf_protect.telemetry.profile_uuid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_remove_profile_display_name_to_jamf_protect_telemetry_profile_display_name_2cb949ad field: jamf_protect.telemetry.event.profile_remove.profile.display_name target_field: jamf_protect.telemetry.profile_display_name type: string ignore_missing: true - script: + tag: script_d00ce187 lang: painless params: itemTypeMap: @@ -48,102 +56,124 @@ processors: ctx.jamf_protect.telemetry.profile_install_source = itemTypeString; } - convert: + tag: convert_jamf_protect_telemetry_event_profile_remove_profile_organization_to_jamf_protect_telemetry_profile_organization_48dd2d3b field: jamf_protect.telemetry.event.profile_remove.profile.organization target_field: jamf_protect.telemetry.profile_organization type: string ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_start_time_to_process_start_128a3584 field: jamf_protect.telemetry.event.profile_remove.instigator.start_time target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_remove_instigator_audit_token_egid_to_jamf_protect_telemetry_event_profile_remove_instigator_audit_token_egid_1ae462f6 field: jamf_protect.telemetry.event.profile_remove.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.profile_remove.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_remove_instigator_audit_token_euid_to_jamf_protect_telemetry_event_profile_remove_instigator_audit_token_euid_1e88751e field: jamf_protect.telemetry.event.profile_remove.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.profile_remove.instigator.audit_token.euid type: string ignore_missing: true - append: + tag: append_user_effective_id_131f3fa1 field: user.effective.id value: '{{{jamf_protect.telemetry.event.profile_remove.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.profile_remove?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_audit_token_e_username_to_process_user_name_7bfaad15 field: jamf_protect.telemetry.event.profile_remove.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_is_platform_binary_to_process_platform_binary_ccac3019 field: jamf_protect.telemetry.event.profile_remove.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_is_es_client_to_process_endpoint_security_client_182a0ad9 field: jamf_protect.telemetry.event.profile_remove.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_cdhash_to_process_hash_cdhash_3ff3304c field: jamf_protect.telemetry.event.profile_remove.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_executable_sha1_to_process_hash_sha1_e64697a2 field: jamf_protect.telemetry.event.profile_remove.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_executable_sha256_to_process_hash_sha256_74ab999a field: jamf_protect.telemetry.event.profile_remove.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_tty_path_to_jamf_protect_telemetry_tty_d984eed5 field: jamf_protect.telemetry.event.profile_remove.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_53ccb468 field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.profile_remove?.instigator?.tty != null - set: + tag: set_process_interactive_9e605973 field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.profile_remove?.instigator?.tty == null - convert: + tag: convert_jamf_protect_telemetry_event_profile_remove_instigator_audit_token_pid_to_process_pid_a71aa517 field: jamf_protect.telemetry.event.profile_remove.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_01368b1a field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_audit_token_uuid_to_process_entity_id_93666fb1 field: jamf_protect.telemetry.event.profile_remove.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_executable_path_to_process_executable_3820035f field: jamf_protect.telemetry.event.profile_remove.instigator.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -155,102 +185,123 @@ processors: } } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_db_path_to_process_working_directory_b57ce837 field: jamf_protect.telemetry.event.profile_remove.db_path target_field: process.working_directory ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.profile_remove?.db_path != null - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_signing_id_to_process_code_signature_signing_id_85acb874 field: jamf_protect.telemetry.event.profile_remove.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_team_id_to_process_code_signature_team_id_b4aed0da field: jamf_protect.telemetry.event.profile_remove.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_codesigning_flags_to_process_code_signature_flags_8aed201f field: jamf_protect.telemetry.event.profile_remove.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_parent_audit_token_uuid_to_process_parent_entity_id_d3787ab4 field: jamf_protect.telemetry.event.profile_remove.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_parent_audit_token_pid_to_process_parent_pid_3f488c32 field: jamf_protect.telemetry.event.profile_remove.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_remove_instigator_parent_audit_token_euid_to_process_parent_user_id_2ec1bac3 field: jamf_protect.telemetry.event.profile_remove.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_parent_audit_token_e_username_to_process_parent_user_name_382bc956 field: jamf_protect.telemetry.event.profile_remove.instigator.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_remove_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_1003a425 field: jamf_protect.telemetry.event.profile_remove.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_remove_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_21a2e153 field: jamf_protect.telemetry.event.profile_remove.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_cf1e0d6c field: jamf_protect.telemetry.event.profile_remove.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_parent_audit_token_exec_path_to_process_parent_executable_da0c48f7 field: jamf_protect.telemetry.event.profile_remove.instigator.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_f21a3d4a field: jamf_protect.telemetry.event.profile_remove.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_responsible_audit_token_pid_to_process_responsible_pid_9f771764 field: jamf_protect.telemetry.event.profile_remove.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_remove_instigator_responsible_audit_token_euid_to_process_responsible_user_id_993e8d2d field: jamf_protect.telemetry.event.profile_remove.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_responsible_audit_token_e_username_to_process_responsible_user_name_aeb3340e field: jamf_protect.telemetry.event.profile_remove.instigator.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_remove_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_c9316a41 field: jamf_protect.telemetry.event.profile_remove.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_profile_remove_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_b3065557 field: jamf_protect.telemetry.event.profile_remove.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_7cbda396 field: jamf_protect.telemetry.event.profile_remove.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_profile_remove_instigator_responsible_audit_token_exec_path_to_process_responsible_executable_ddbd18e5 field: jamf_protect.telemetry.event.profile_remove.instigator.responsible_audit_token.exec_path target_field: process.responsible.executable - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_pty_close.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_pty_close.yml index 1bb85116c14..ac27c03eecb 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_pty_close.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_pty_close.yml @@ -2,21 +2,25 @@ description: Pipeline for parsing specific fields related to pty close events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_fbb2df5a field: event.reason value: A pseudoterminal control device is closed - append: + tag: append_event_type_90fe2b35 field: event.type value: end - rename: + tag: rename_jamf_protect_telemetry_event_pty_close_dev_to_jamf_protect_telemetry_dev_9f29f174 field: jamf_protect.telemetry.event.pty_close.dev target_field: jamf_protect.telemetry.dev ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - pipeline: + tag: pipeline_0d75fa7a name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_pty_grant.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_pty_grant.yml index 800d2fc74ba..17ad60f7029 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_pty_grant.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_pty_grant.yml @@ -2,21 +2,25 @@ description: Pipeline for parsing specific fields related to pty grant events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_bd9ce7fb field: event.reason value: A pseudoterminal control device is granted - append: + tag: append_event_type_fffb7a5a field: event.type value: start - rename: + tag: rename_jamf_protect_telemetry_event_pty_grant_dev_to_jamf_protect_telemetry_dev_debcecde field: jamf_protect.telemetry.event.pty_grant.dev target_field: jamf_protect.telemetry.dev ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - pipeline: + tag: pipeline_0d75fa7a name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_remount.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_remount.yml index a32551b4915..2fbdec2374c 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_remount.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_remount.yml @@ -2,69 +2,85 @@ description: Pipeline for parsing specific fields related to vollume remount events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - append: + tag: append_event_type_fffb7a5a field: event.type value: start - set: + tag: set_event_reason_374d23fd field: event.reason value: A file system has been remounted - rename: + tag: rename_jamf_protect_telemetry_event_remount_statfs_f_mntfromname_to_volume_device_name_03705701 field: jamf_protect.telemetry.event.remount.statfs.f_mntfromname target_field: volume.device_name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_remount_statfs_f_mntonname_to_volume_mount_name_1ff0885d field: jamf_protect.telemetry.event.remount.statfs.f_mntonname target_field: volume.mount_name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_remount_statfs_f_fstypename_to_volume_file_system_type_1b5820c8 field: jamf_protect.telemetry.event.remount.statfs.f_fstypename target_field: volume.file_system_type ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_remount_device_size_to_volume_size_469b4289 field: jamf_protect.telemetry.event.remount.device.size target_field: volume.size ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_remount_device_bus_name_to_volume_bus_type_98b4c842 field: jamf_protect.telemetry.event.remount.device.bus_name target_field: volume.bus_type ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_remount_device_product_id_to_volume_product_id_c5042c69 field: jamf_protect.telemetry.event.remount.device.product_id target_field: volume.product_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_remount_device_product_name_to_volume_product_name_52d57ef9 field: jamf_protect.telemetry.event.remount.device.product_name target_field: volume.product_name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_remount_device_vendor_id_to_volume_vendor_id_5f9f1b7d field: jamf_protect.telemetry.event.remount.device.vendor_id target_field: volume.vendor_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_remount_device_vendor_name_to_volume_vendor_name_a0136971 field: jamf_protect.telemetry.event.remount.device.vendor_name target_field: volume.vendor_name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_remount_device_protocol_to_volume_device_type_d89b8db4 field: jamf_protect.telemetry.event.remount.device.protocol target_field: volume.device_type ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_remount_device_is_removable_to_volume_removable_d9e90b90 field: jamf_protect.telemetry.event.remount.device.is_removable target_field: volume.removable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_remount_device_is_writable_to_volume_writable_3a647e96 field: jamf_protect.telemetry.event.remount.device.is_writable target_field: volume.writable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_remount_device_serial_number_to_volume_serial_number_4afd65bd field: jamf_protect.telemetry.event.remount.device.serial_number target_field: volume.serial_number ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - pipeline: - name: '{{ IngestPipeline "pipeline_object_process" }}' \ No newline at end of file + tag: pipeline_0d75fa7a + name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_screensharing_attach.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_screensharing_attach.yml index 16681f2fbd6..84a57ab4b26 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_screensharing_attach.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_screensharing_attach.yml @@ -2,19 +2,23 @@ description: Pipeline for parsing specific fields related to screenscharing attached events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_b314cc48 field: event.reason value: A screen sharing session has attached to a graphical session - append: + tag: append_event_type_fffb7a5a field: event.type value: start - append: + tag: append_event_category_d21db9a2 field: event.category - value: session + value: session - script: + tag: script_c23eceb5 lang: painless source: | ctx.event = ctx.event != null ? ctx.event : new HashMap(); @@ -29,27 +33,32 @@ processors: ctx.event.outcome = 'unknown'; } - convert: + tag: convert_jamf_protect_telemetry_event_screensharing_attach_existing_session_to_jamf_protect_telemetry_existing_session_cbd355a4 field: jamf_protect.telemetry.event.screensharing_attach.existing_session target_field: jamf_protect.telemetry.existing_session type: boolean ignore_missing: true ignore_failure: true - convert: + tag: convert_jamf_protect_telemetry_event_screensharing_attach_graphical_session_id_to_jamf_protect_telemetry_graphical_authentication_username_34566dc5 field: jamf_protect.telemetry.event.screensharing_attach.graphical_session_id target_field: jamf_protect.telemetry.graphical_authentication_username type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_screensharing_attach_authentication_username_to_jamf_protect_telemetry_graphical_authentication_username_b616f5da field: jamf_protect.telemetry.event.screensharing_attach.authentication_username target_field: jamf_protect.telemetry.graphical_authentication_username type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_screensharing_attach_session_username_to_jamf_protect_telemetry_session_username_6910c1d2 field: jamf_protect.telemetry.event.screensharing_attach.session_username target_field: jamf_protect.telemetry.session_username type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_screensharing_attach_viewer_appleid_to_jamf_protect_viewer_appleid_6a19672f field: jamf_protect.telemetry.event.screensharing_attach.viewer_appleid target_field: jamf_protect.viewer_appleid type: string @@ -57,30 +66,36 @@ processors: ignore_missing: true ignore_failure: true - convert: + tag: convert_jamf_protect_telemetry_event_screensharing_attach_authentication_type_to_jamf_protect_telemetry_authentication_type_c8697154 field: jamf_protect.telemetry.event.screensharing_attach.authentication_type target_field: jamf_protect.telemetry.authentication_type type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_screensharing_attach_authentication_type_to_jamf_protect_telemetry_authentication_type_c8697154 field: jamf_protect.telemetry.event.screensharing_attach.authentication_type target_field: jamf_protect.telemetry.authentication_type type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_screensharing_attach_source_address_to_source_ip_d8349a03 field: jamf_protect.telemetry.event.screensharing_attach.source_address target_field: source.ip ignore_missing: true - append: + tag: append_related_user_220df46c field: related.user value: '{{{jamf_protect.telemetry.session_username}}}' if: ctx.jamf_protect?.session_username != null allow_duplicates: false - append: + tag: append_related_user_7c102396 field: related.user value: '{{{jamf_protect.telemetry.graphical_authentication_username}}}' if: ctx.jamf_protect?.graphical_authentication_username != null allow_duplicates: false - script: + tag: script_aaf6e222 lang: painless params: itemTypeMap: @@ -95,8 +110,9 @@ processors: ctx.jamf_protect = ctx.jamf_protect != null ? ctx.jamf_protect : new HashMap(); ctx.jamf_protect.telemetry.source_address_type = itemTypeString; } -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - pipeline: - name: '{{ IngestPipeline "pipeline_object_process" }}' \ No newline at end of file + tag: pipeline_0d75fa7a + name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_screensharing_detach.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_screensharing_detach.yml index 20a28b8232d..7189aa90f5e 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_screensharing_detach.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_screensharing_detach.yml @@ -2,19 +2,23 @@ description: Pipeline for parsing specific fields related to screensharing detached events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_41103f0d field: event.reason value: A screen sharing session has detached from a graphical session - append: + tag: append_event_type_90fe2b35 field: event.type value: end - append: + tag: append_event_category_d21db9a2 field: event.category value: session - script: + tag: script_5ae318d5 lang: painless source: | ctx.event = ctx.event != null ? ctx.event : new HashMap(); @@ -29,11 +33,13 @@ processors: ctx.event.outcome = 'unknown'; } - convert: + tag: convert_jamf_protect_telemetry_event_screensharing_detach_graphical_session_id_to_jamf_protect_telemetry_graphical_authentication_username_847d25e7 field: jamf_protect.telemetry.event.screensharing_detach.graphical_session_id target_field: jamf_protect.telemetry.graphical_authentication_username type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_screensharing_detach_viewer_appleid_to_jamf_protect_viewer_appleid_63896993 field: jamf_protect.telemetry.event.screensharing_detach.viewer_appleid target_field: jamf_protect.viewer_appleid type: string @@ -41,10 +47,12 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_screensharing_detach_source_address_to_source_ip_9bfb67c1 field: jamf_protect.telemetry.event.screensharing_detach.source_address target_field: source.ip ignore_missing: true - script: + tag: script_df68d29b lang: painless params: itemTypeMap: @@ -59,8 +67,9 @@ processors: ctx.jamf_protect = ctx.jamf_protect != null ? ctx.jamf_protect : new HashMap(); ctx.jamf_protect.telemetry.source_address_type = itemTypeString; } -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - pipeline: + tag: pipeline_0d75fa7a name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_settime.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_settime.yml index 58c5bfd3812..da292a0b9cc 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_settime.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_settime.yml @@ -2,20 +2,24 @@ description: Pipeline for parsing specific fields related to settime events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_3bdc5e20 field: event.reason value: The system time was attempted to be set - append: + tag: append_event_type_0602e5be field: event.type value: change - append: + tag: append_event_category_1c667e2c field: event.category - value: configuration -########################## -## ECS Process ## -########################## + value: configuration + ########################## + ## ECS Process ## + ########################## - pipeline: + tag: pipeline_0d75fa7a name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_su.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_su.yml index dd0b2cdeadd..8573650509b 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_su.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_su.yml @@ -2,19 +2,23 @@ description: Pipeline for parsing specific fields related to su events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_17d5df9e field: event.reason value: A user attempts to start a new shell using a substitute user identity - append: + tag: append_event_type_fffb7a5a field: event.type value: start - append: + tag: append_event_category_109b4d72 field: event.category value: authentication - script: + tag: script_b92a9dcd lang: painless source: > ctx.event = ctx.event != null ? ctx.event : new HashMap(); @@ -29,20 +33,24 @@ processors: ctx.event.outcome = 'unknown'; } - rename: + tag: rename_jamf_protect_telemetry_event_su_username_to_user_name_db45debd field: jamf_protect.telemetry.event.su.username target_field: user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_su_uid_to_user_id_3d8802df field: jamf_protect.telemetry.event.su.uid target_field: user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_su_argv_to_process_args_ee94e293 field: jamf_protect.telemetry.event.su.argv target_field: process.args ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.su?.argv != null - script: + tag: script_9e475c50 description: Counts the indexes of the arguments lang: painless source: > @@ -54,47 +62,56 @@ processors: } ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_su_env_to_process_env_vars_d879fc62 field: jamf_protect.telemetry.event.su.env target_field: process.env_vars ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.su?.env != null - convert: + tag: convert_jamf_protect_telemetry_event_su_env_count_to_jamf_protect_telemetry_env_count_99a0b268 field: jamf_protect.telemetry.event.su.env_count target_field: jamf_protect.telemetry.env_count type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_su_shell_to_jamf_protect_telemetry_shell_e626b83c field: jamf_protect.telemetry.event.su.shell target_field: jamf_protect.telemetry.shell type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_su_from_username_to_jamf_protect_telemetry_from_username_24c4f274 field: jamf_protect.telemetry.event.su.from_username target_field: jamf_protect.telemetry.from_username type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_su_to_username_to_jamf_protect_telemetry_to_username_3dafafc4 field: jamf_protect.telemetry.event.su.to_username target_field: jamf_protect.telemetry.to_username type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_su_failure_message_to_jamf_protect_failure_reason_add5b330 field: jamf_protect.telemetry.event.su.failure_message target_field: jamf_protect.failure_reason type: string ignore_missing: true - append: + tag: append_related_user_4c442fde field: related.user value: '{{{jamf_protect.telemetry.to_username}}}' if: ctx.jamf_protect?.to_username != null allow_duplicates: false - append: + tag: append_related_user_259f8822 field: related.user value: '{{{jamf_protect.telemetry.from_username}}}' if: ctx.jamf_protect?.from_username != null allow_duplicates: false -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - pipeline: - name: '{{ IngestPipeline "pipeline_object_process" }}' \ No newline at end of file + tag: pipeline_0d75fa7a + name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_sudo.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_sudo.yml index ee936ddcfb7..6ff08d92cad 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_sudo.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_sudo.yml @@ -2,31 +2,37 @@ description: Pipeline for parsing specific fields related to sudo events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_86945cd2 field: event.reason value: A sudo attempt occurred - append: + tag: append_event_type_fffb7a5a field: event.type value: start - rename: + tag: rename_jamf_protect_telemetry_event_sudo_command_to_process_command_line_32d9e640 field: jamf_protect.telemetry.event.sudo.command target_field: process.command_line ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_sudo_attribute_name_to_jamf_protect_telemetry_attribute_name_39101063 field: jamf_protect.telemetry.event.sudo.attribute_name target_field: jamf_protect.telemetry.attribute_name type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_sudo_attribute_value_to_jamf_protect_telemetry_attribute_value_0702a9b7 field: jamf_protect.telemetry.event.sudo.attribute_value target_field: jamf_protect.telemetry.attribute_value type: string ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - pipeline: + tag: pipeline_0d75fa7a name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_system_performance.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_system_performance.yml index 0e1dfbfd7c4..08996ebf691 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_system_performance.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_system_performance.yml @@ -2,20 +2,27 @@ description: Pipeline for Jamf Protect Telemetry system_performance events. processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_823ba76c field: event.reason value: Collection of system system_performance data - append: + tag: append_event_category_3fa422a0 field: event.category value: host - rename: + tag: rename_jamf_protect_telemetry_event_system_performance_metrics_tasks_to_jamf_protect_telemetry_system_performance_489cb3f5 field: jamf_protect.telemetry.event.system_performance.metrics.tasks target_field: jamf_protect.telemetry.system_performance ignore_missing: true on_failure: +- append: + field: tags + value: preserve_original_event + allow_duplicates: false - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_tcc_modify.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_tcc_modify.yml index ce2515b4a1a..65e369837a4 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_tcc_modify.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_tcc_modify.yml @@ -2,31 +2,38 @@ description: Pipeline for parsing specific fields related to TCC Permission events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_23cf90bd field: event.reason value: A Transparency Consent and Control (TCC) permission is granted or revoked. - append: + tag: append_event_type_0602e5be field: event.type value: change - append: + tag: append_event_category_1c667e2c field: event.category value: configuration - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_service_to_jamf_protect_telemetry_tcc_service_1fe3b8d8 field: jamf_protect.telemetry.event.tcc_modify.service target_field: jamf_protect.telemetry.tcc_service ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_identity_to_jamf_protect_telemetry_tcc_identity_cfa160ee field: jamf_protect.telemetry.event.tcc_modify.identity target_field: jamf_protect.telemetry.tcc_identity ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_identity_type_to_jamf_protect_telemetry_tcc_identity_type_69aa9fa8 field: jamf_protect.telemetry.event.tcc_modify.identity_type target_field: jamf_protect.telemetry.tcc_identity_type ignore_missing: true - script: + tag: script_1badeff6 lang: painless params: itemTypeMap: @@ -45,6 +52,7 @@ processors: ctx.jamf_protect.telemetry.tcc_right = itemTypeString; } - script: + tag: script_c46dbeb1 lang: painless params: itemTypeMap: @@ -70,6 +78,7 @@ processors: ctx.jamf_protect.telemetry.tcc_reason = itemTypeString; } - script: + tag: script_dbcd9aaa lang: painless params: itemTypeMap: @@ -84,98 +93,119 @@ processors: ctx.jamf_protect = ctx.jamf_protect != null ? ctx.jamf_protect : new HashMap(); ctx.jamf_protect.telemetry.tcc_update_type = itemTypeString; } -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_start_time_to_process_start_b26fb93b field: jamf_protect.telemetry.event.tcc_modify.instigator.start_time target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_tcc_modify_instigator_audit_token_egid_to_jamf_protect_telemetry_event_tcc_modify_instigator_audit_token_egid_5b26a3d4 field: jamf_protect.telemetry.event.tcc_modify.instigator.audit_token.egid target_field: jamf_protect.telemetry.event.tcc_modify.instigator.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_tcc_modify_instigator_audit_token_euid_to_jamf_protect_telemetry_event_tcc_modify_instigator_audit_token_euid_0a5bc148 field: jamf_protect.telemetry.event.tcc_modify.instigator.audit_token.euid target_field: jamf_protect.telemetry.event.tcc_modify.instigator.audit_token.euid type: string ignore_missing: true - append: + tag: append_user_effective_id_8e9ef011 field: user.effective.id value: '{{{jamf_protect.telemetry.event.tcc_modify.instigator.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.event?.tcc_modify?.instigator?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_audit_token_e_username_to_process_user_name_171ebbd2 field: jamf_protect.telemetry.event.tcc_modify.instigator.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_is_platform_binary_to_process_platform_binary_a55c62a2 field: jamf_protect.telemetry.event.tcc_modify.instigator.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_is_es_client_to_process_endpoint_security_client_a9dea928 field: jamf_protect.telemetry.event.tcc_modify.instigator.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_cdhash_to_process_hash_cdhash_759c305f field: jamf_protect.telemetry.event.tcc_modify.instigator.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_executable_sha1_to_process_hash_sha1_eb076b3f field: jamf_protect.telemetry.event.tcc_modify.instigator.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_executable_sha256_to_process_hash_sha256_36335077 field: jamf_protect.telemetry.event.tcc_modify.instigator.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_tty_path_to_jamf_protect_telemetry_tty_756df41c field: jamf_protect.telemetry.event.tcc_modify.instigator.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_a97720bd field: process.interactive value: true if: ctx.jamf_protect.telemetry?.event?.tcc_modify?.instigator?.tty != null - set: + tag: set_process_interactive_4f78fef8 field: process.interactive value: false if: ctx.jamf_protect.telemetry?.event?.tcc_modify?.instigator?.tty == null - convert: + tag: convert_jamf_protect_telemetry_event_tcc_modify_instigator_audit_token_pid_to_process_pid_669c6e5c field: jamf_protect.telemetry.event.tcc_modify.instigator.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_1a285579 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_audit_token_uuid_to_process_entity_id_b5f9a4c2 field: jamf_protect.telemetry.event.tcc_modify.instigator.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_executable_path_to_process_executable_1aac0d0c field: jamf_protect.telemetry.event.tcc_modify.instigator.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -187,102 +217,123 @@ processors: } } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_db_path_to_process_working_directory_31316b9f field: jamf_protect.telemetry.event.tcc_modify.db_path target_field: process.working_directory ignore_missing: true if: ctx.jamf_protect?.telemetry?.event?.tcc_modify?.db_path != null - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_signing_id_to_process_code_signature_signing_id_6f68c37b field: jamf_protect.telemetry.event.tcc_modify.instigator.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_team_id_to_process_code_signature_team_id_18b57835 field: jamf_protect.telemetry.event.tcc_modify.instigator.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_codesigning_flags_to_process_code_signature_flags_e577bf28 field: jamf_protect.telemetry.event.tcc_modify.instigator.codesigning_flags target_field: process.code_signature.flags ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_parent_audit_token_uuid_to_process_parent_entity_id_c1169307 field: jamf_protect.telemetry.event.tcc_modify.instigator.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_parent_audit_token_pid_to_process_parent_pid_aca41ad3 field: jamf_protect.telemetry.event.tcc_modify.instigator.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_tcc_modify_instigator_parent_audit_token_euid_to_process_parent_user_id_4c5d72b0 field: jamf_protect.telemetry.event.tcc_modify.instigator.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_parent_audit_token_e_username_to_process_parent_user_name_7fd5cce9 field: jamf_protect.telemetry.event.tcc_modify.instigator.parent_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_tcc_modify_instigator_parent_audit_token_ruid_to_process_parent_real_user_id_243b6104 field: jamf_protect.telemetry.event.tcc_modify.instigator.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_tcc_modify_instigator_parent_audit_token_rgid_to_process_parent_real_group_id_3da27d3c field: jamf_protect.telemetry.event.tcc_modify.instigator.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_eae14f17 field: jamf_protect.telemetry.event.tcc_modify.instigator.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_parent_audit_token_exec_path_to_process_parent_executable_856ec8f8 field: jamf_protect.telemetry.event.tcc_modify.instigator.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_responsible_audit_token_uuid_to_process_responsible_entity_id_98a71b19 field: jamf_protect.telemetry.event.tcc_modify.instigator.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_responsible_audit_token_pid_to_process_responsible_pid_49353a81 field: jamf_protect.telemetry.event.tcc_modify.instigator.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_tcc_modify_instigator_responsible_audit_token_euid_to_process_responsible_user_id_2503340e field: jamf_protect.telemetry.event.tcc_modify.instigator.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_responsible_audit_token_e_username_to_process_responsible_user_name_b7848db1 field: jamf_protect.telemetry.event.tcc_modify.instigator.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_tcc_modify_instigator_responsible_audit_token_ruid_to_process_responsible_real_user_id_b4954ac4 field: jamf_protect.telemetry.event.tcc_modify.instigator.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_event_tcc_modify_instigator_responsible_audit_token_rgid_to_process_responsible_real_group_id_85dec570 field: jamf_protect.telemetry.event.tcc_modify.instigator.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_d22a456d field: jamf_protect.telemetry.event.tcc_modify.instigator.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_tcc_modify_instigator_responsible_audit_token_exec_path_to_process_responsible_executable_d65e0fe2 field: jamf_protect.telemetry.event.tcc_modify.instigator.responsible_audit_token.exec_path target_field: process.responsible.executable - ignore_missing: true \ No newline at end of file + ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_unmount.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_unmount.yml index 05b7d471a95..0e06dcae893 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_unmount.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_unmount.yml @@ -2,69 +2,85 @@ description: Pipeline for parsing specific fields related to volume unmount events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_9f2554b3 field: event.reason value: A file system has been unmounted - append: + tag: append_event_type_90fe2b35 field: event.type value: end - rename: + tag: rename_jamf_protect_telemetry_event_unmount_statfs_f_mntfromname_to_volume_device_name_511b9aff field: jamf_protect.telemetry.event.unmount.statfs.f_mntfromname target_field: volume.device_name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_unmount_statfs_f_mntonname_to_volume_mount_name_5cd454b3 field: jamf_protect.telemetry.event.unmount.statfs.f_mntonname target_field: volume.mount_name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_unmount_statfs_f_fstypename_to_volume_file_system_type_a06088f2 field: jamf_protect.telemetry.event.unmount.statfs.f_fstypename target_field: volume.file_system_type ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_unmount_device_size_to_volume_size_3329e9bf field: jamf_protect.telemetry.event.unmount.device.size target_field: volume.size ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_unmount_device_bus_name_to_volume_bus_type_721754ec field: jamf_protect.telemetry.event.unmount.device.bus_name target_field: volume.bus_type ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_unmount_device_product_id_to_volume_product_id_153510af field: jamf_protect.telemetry.event.unmount.device.product_id target_field: volume.product_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_unmount_device_product_name_to_volume_product_name_81f054ef field: jamf_protect.telemetry.event.unmount.device.product_name target_field: volume.product_name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_unmount_device_vendor_id_to_volume_vendor_id_b959e123 field: jamf_protect.telemetry.event.unmount.device.vendor_id target_field: volume.vendor_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_unmount_device_vendor_name_to_volume_vendor_name_635cc6af field: jamf_protect.telemetry.event.unmount.device.vendor_name target_field: volume.vendor_name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_unmount_device_protocol_to_volume_device_type_bff9c166 field: jamf_protect.telemetry.event.unmount.device.protocol target_field: volume.device_type ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_unmount_device_is_removable_to_volume_removable_2007a652 field: jamf_protect.telemetry.event.unmount.device.is_removable target_field: volume.removable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_unmount_device_is_writable_to_volume_writable_4a2f67f0 field: jamf_protect.telemetry.event.unmount.device.is_writable target_field: volume.writable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_unmount_device_serial_number_to_volume_serial_number_4460458b field: jamf_protect.telemetry.event.unmount.device.serial_number target_field: volume.serial_number ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - pipeline: - name: '{{ IngestPipeline "pipeline_object_process" }}' \ No newline at end of file + tag: pipeline_0d75fa7a + name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_xp_malware_detected.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_xp_malware_detected.yml index e910d23b89a..d739b706cc5 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_xp_malware_detected.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_xp_malware_detected.yml @@ -2,28 +2,34 @@ description: Pipeline for parsing specific fields related to xprotect malware detected events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_845e2755 field: event.reason value: Apple’s XProtect detected malware on the system - append: + tag: append_event_category_425d1f27 field: event.category - value: malware + value: malware - rename: + tag: rename_jamf_protect_telemetry_event_xp_malware_detected_detected_path_to_threat_indicator_file_path_f986a31c field: jamf_protect.telemetry.event.xp_malware_detected.detected_path target_field: threat.indicator.file.path ignore_missing: true - set: + tag: set_threat_indicator_type_e912359f field: threat.indicator.type value: file - rename: + tag: rename_jamf_protect_telemetry_event_xp_malware_detected_malware_identifier_to_rule_name_2019442a field: jamf_protect.telemetry.event.xp_malware_detected.malware_identifier target_field: rule.name ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - pipeline: + tag: pipeline_0d75fa7a name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_xp_malware_remediated.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_xp_malware_remediated.yml index 5e8804e3c9c..797adde9b05 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_xp_malware_remediated.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_event_xp_malware_remediated.yml @@ -2,16 +2,19 @@ description: Pipeline for parsing specific fields related to xprotect malware remediated events in Jamf Protect processors: -########################## -## ECS Event Specific ## -########################## + ########################## + ## ECS Event Specific ## + ########################## - set: + tag: set_event_reason_4d0ee003 field: event.reason value: Apple’s XProtect remediated malware on the system - append: + tag: append_event_category_425d1f27 field: event.category - value: malware + value: malware - script: + tag: script_93b72c51 lang: painless source: > ctx.event = ctx.event != null ? ctx.event : new HashMap(); @@ -26,22 +29,27 @@ processors: ctx.event.outcome = 'unknown'; } - rename: + tag: rename_jamf_protect_telemetry_event_xp_malware_remediated_remediated_path_to_threat_indicator_file_path_c7526564 field: jamf_protect.telemetry.event.xp_malware_remediated.remediated_path target_field: threat.indicator.file.path ignore_missing: true - set: + tag: set_threat_indicator_type_e912359f field: threat.indicator.type value: file - rename: + tag: rename_jamf_protect_telemetry_event_xp_malware_remediated_malware_identifier_to_rule_name_79522ff8 field: jamf_protect.telemetry.event.xp_malware_remediated.malware_identifier target_field: rule.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_event_xp_malware_remediated_signature_version_to_rule_version_6ca769db field: jamf_protect.telemetry.event.xp_malware_remediated.signature_version target_field: rule.version ignore_missing: true -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - pipeline: + tag: pipeline_0d75fa7a name: '{{ IngestPipeline "pipeline_object_process" }}' diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_instigator_object.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_instigator_object.yml index 8cb9345a186..35fafc1ef03 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_instigator_object.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_instigator_object.yml @@ -1,81 +1,95 @@ --- description: Pipeline for Jamf instigator_object audit logs. processors: -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_process_start_time_to_process_start_b14fb885 field: jamf_protect.telemetry.process.start_time target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_process_audit_token_egid_to_jamf_protect_telemetry_process_audit_token_egid_9112d1eb field: jamf_protect.telemetry.process.audit_token.egid target_field: jamf_protect.telemetry.process.audit_token.egid type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_jamf_protect_telemetry_process_audit_token_euid_to_jamf_protect_telemetry_process_audit_token_euid_920d2257 field: jamf_protect.telemetry.process.audit_token.euid target_field: jamf_protect.telemetry.process.audit_token.euid type: string ignore_missing: true ignore_failure: true - append: + tag: append_user_effective_id_7b6ab2ef field: user.effective.id value: '{{{jamf_protect.telemetry.process.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.process?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_process_executable_sha1_to_process_hash_sha1_e7ef4185 field: jamf_protect.telemetry.process.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_process_executable_sha256_to_process_hash_sha256_6f5330fd field: jamf_protect.telemetry.process.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_custom_tty_path_to_jamf_protect_telemetry_tty_63e85edc field: jamf_protect.telemetry.custom.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_638b3399 field: process.interactive value: true if: ctx.jamf_protect.telemetry?.process?.tty != null - append: + tag: append_related_hash_4075cce6 field: related.hash value: '{{{jamf_protect.telemetry.process.executable.sha1}}}' if: ctx.jamf_protect.telemetry?.process?.instigator?.executable?.sha1 != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_jamf_protect_telemetry_process_audit_token_pid_to_process_pid_e7122b56 field: jamf_protect.telemetry.process.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_eb9e5c4f field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_process_audit_token_uuid_to_process_entity_id_a9a0aecc field: jamf_protect.telemetry.process.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_process_executable_path_to_process_executable_c80a04f6 field: jamf_protect.telemetry.process.executable.path target_field: process.executable ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true -########################## -## ECS Code Signing ## -########################## - + ########################## + ## ECS Code Signing ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_process_signing_id_to_process_code_signature_signing_id_9eefc109 field: jamf_protect.telemetry.process.signing_id target_field: process.code_signature.signing_id ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_object_process.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_object_process.yml index 778e5468520..98434255b5f 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_object_process.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/pipeline_object_process.yml @@ -2,98 +2,119 @@ description: Pipeline for parsing process objects in specific Jamf Protect Telemetry events. This pipeline is called in events where there is no instigator existing and therefore parsing the process object itself. processors: -########################## -## ECS Process ## -########################## + ########################## + ## ECS Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_process_start_time_to_process_start_b14fb885 field: jamf_protect.telemetry.process.start_time target_field: process.start ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_process_audit_token_egid_to_jamf_protect_telemetry_process_audit_token_egid_f0c8cda0 field: jamf_protect.telemetry.process.audit_token.egid target_field: jamf_protect.telemetry.process.audit_token.egid type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_process_audit_token_euid_to_jamf_protect_telemetry_process_audit_token_euid_54fc53bc field: jamf_protect.telemetry.process.audit_token.euid target_field: jamf_protect.telemetry.process.audit_token.euid type: string ignore_missing: true - append: + tag: append_user_effective_id_7b6ab2ef field: user.effective.id value: '{{{jamf_protect.telemetry.process.audit_token.euid}}}' if: ctx.jamf_protect.telemetry?.process?.audit_token?.euid != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_jamf_protect_telemetry_process_audit_token_e_username_to_process_user_name_1028795c field: jamf_protect.telemetry.process.audit_token.e_username target_field: process.user.name ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_process_is_platform_binary_to_process_platform_binary_b4974ff8 field: jamf_protect.telemetry.process.is_platform_binary target_field: process.platform_binary ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_process_is_es_client_to_process_endpoint_security_client_8b090f5a field: jamf_protect.telemetry.process.is_es_client target_field: process.endpoint_security_client ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_process_cdhash_to_process_hash_cdhash_236972ed field: jamf_protect.telemetry.process.cdhash target_field: process.hash.cdhash ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_process_executable_sha1_to_process_hash_sha1_e7ef4185 field: jamf_protect.telemetry.process.executable.sha1 target_field: process.hash.sha1 ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_process_executable_sha256_to_process_hash_sha256_6f5330fd field: jamf_protect.telemetry.process.executable.sha256 target_field: process.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' if: ctx.process?.hash?.sha256 != null allow_duplicates: false - append: + tag: append_related_hash_f4079ea4 field: related.hash value: '{{{process.hash.cdhash}}}' if: ctx.process?.hash?.cdhash != null && ctx.process.hash.cdhash != '' allow_duplicates: false - rename: + tag: rename_jamf_protect_telemetry_custom_tty_path_to_jamf_protect_telemetry_tty_63e85edc field: jamf_protect.telemetry.custom.tty.path target_field: jamf_protect.telemetry.tty ignore_missing: true - set: + tag: set_process_interactive_638b3399 field: process.interactive value: true if: ctx.jamf_protect.telemetry?.process?.tty != null - set: + tag: set_process_interactive_cc1f1d24 field: process.interactive value: false if: ctx.jamf_protect.telemetry?.process?.tty == null - convert: + tag: convert_jamf_protect_telemetry_process_audit_token_pid_to_process_pid_e7122b56 field: jamf_protect.telemetry.process.audit_token.pid target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_eb9e5c4f field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_jamf_protect_telemetry_process_audit_token_uuid_to_process_entity_id_a9a0aecc field: jamf_protect.telemetry.process.audit_token.uuid target_field: process.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_process_executable_path_to_process_executable_c80a04f6 field: jamf_protect.telemetry.process.executable.path target_field: process.executable ignore_missing: true - script: + tag: script_f232bce3 lang: painless source: > if (ctx.process?.executable != null) { @@ -105,99 +126,118 @@ processors: } } - rename: + tag: rename_jamf_protect_telemetry_thread_thread_id_to_process_thread_id_3987a29f field: jamf_protect.telemetry.thread.thread_id target_field: process.thread.id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_process_signing_id_to_process_code_signature_signing_id_9eefc109 field: jamf_protect.telemetry.process.signing_id target_field: process.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_process_team_id_to_process_code_signature_team_id_9e6cbef3 field: jamf_protect.telemetry.process.team_id target_field: process.code_signature.team_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_process_codesigning_flags_to_process_code_signature_flags_6fc05176 field: jamf_protect.telemetry.process.codesigning_flags target_field: process.code_signature.flags ignore_missing: true -########################## -## ECS Parent Process ## -########################## + ########################## + ## ECS Parent Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_process_parent_audit_token_uuid_to_process_parent_entity_id_e0b5b269 field: jamf_protect.telemetry.process.parent_audit_token.uuid target_field: process.parent.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_process_parent_audit_token_pid_to_process_parent_pid_09005669 field: jamf_protect.telemetry.process.parent_audit_token.pid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_process_parent_audit_token_euid_to_process_parent_user_id_54ade22a field: jamf_protect.telemetry.process.parent_audit_token.euid target_field: process.parent.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_process_responsible_audit_token_e_username_to_process_parent_user_name_89a01351 field: jamf_protect.telemetry.process.responsible_audit_token.e_username target_field: process.parent.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_process_parent_audit_token_ruid_to_process_parent_real_user_id_cdaf1096 field: jamf_protect.telemetry.process.parent_audit_token.ruid target_field: process.parent.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_process_parent_audit_token_rgid_to_process_parent_real_group_id_8ee2dd52 field: jamf_protect.telemetry.process.parent_audit_token.rgid target_field: process.parent.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_process_parent_audit_token_signing_id_to_process_parent_code_signature_signing_id_68a75ac9 field: jamf_protect.telemetry.process.parent_audit_token.signing_id target_field: process.parent.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_process_parent_audit_token_exec_path_to_process_parent_executable_d827ccea field: jamf_protect.telemetry.process.parent_audit_token.exec_path target_field: process.parent.executable ignore_missing: true -########################## -## ECS Responsible Process ## -########################## - + ########################## + ## ECS Responsible Process ## + ########################## - rename: + tag: rename_jamf_protect_telemetry_process_responsible_audit_token_uuid_to_process_responsible_entity_id_fd2cffa3 field: jamf_protect.telemetry.process.responsible_audit_token.uuid target_field: process.responsible.entity_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_process_responsible_audit_token_pid_to_process_responsible_pid_07ce0267 field: jamf_protect.telemetry.process.responsible_audit_token.pid target_field: process.responsible.pid ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_process_responsible_audit_token_euid_to_process_responsible_user_id_dd1f5998 field: jamf_protect.telemetry.process.responsible_audit_token.euid target_field: process.responsible.user.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_process_responsible_audit_token_e_username_to_process_responsible_user_name_47564223 field: jamf_protect.telemetry.process.responsible_audit_token.e_username target_field: process.responsible.user.name ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_process_responsible_audit_token_ruid_to_process_responsible_real_user_id_9e63753e field: jamf_protect.telemetry.process.responsible_audit_token.ruid target_field: process.responsible.real_user.id type: string ignore_missing: true - convert: + tag: convert_jamf_protect_telemetry_process_responsible_audit_token_rgid_to_process_responsible_real_group_id_6a290676 field: jamf_protect.telemetry.process.responsible_audit_token.rgid target_field: process.responsible.real_group.id type: string ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_process_responsible_audit_token_signing_id_to_process_responsible_code_signature_signing_id_fa554797 field: jamf_protect.telemetry.process.responsible_audit_token.signing_id target_field: process.responsible.code_signature.signing_id ignore_missing: true - rename: + tag: rename_jamf_protect_telemetry_process_responsible_audit_token_exec_path_to_process_responsible_executable_a15031bc field: jamf_protect.telemetry.process.responsible_audit_token.exec_path target_field: process.responsible.executable ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/default.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/default.yml index 2e2f447febd..1efe7ee5b1f 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/default.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/default.yml @@ -2,32 +2,41 @@ description: Pipeline for Jamf Protect Telemetry logs. processors: - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - rename: + tag: rename_message_to_event_original_30bfc018 field: message target_field: event.original ignore_missing: true - json: + tag: json_event_original_to_json_cac66847 field: event.original target_field: json ignore_failure: true - pipeline: + tag: pipeline_7e82d0ee name: '{{ IngestPipeline "pipeline_system_performance_metrics" }}' if: ctx.json?.header?.event_name == 'SYSTEM_PERFORMANCE_METRICS' - pipeline: + tag: pipeline_49135f14 name: '{{ IngestPipeline "pipeline_audit" }}' if: ctx.json?.header?.event_name != null && ctx.json?.header?.event_name.startsWith('AUE_') - pipeline: + tag: pipeline_4ed89c5e name: '{{ IngestPipeline "pipeline_bios_firmware_versions" }}' if: ctx.json?.header?.event_name == 'BIOS_FIRMWARE_VERSIONS' - pipeline: + tag: pipeline_b277b479 name: '{{ IngestPipeline "pipeline_event" }}' if: "['FILE_COLLECTION_EVENT','PLAINTEXT_LOG_COLLECTION'].contains(ctx.json?.header?.event_name)" - remove: + tag: remove_json_29cdffdc field: json ignore_missing: true - script: + tag: script_e2ccbdd7 description: Drops null/empty values recursively. lang: painless source: | @@ -48,3 +57,11 @@ on_failure: - set: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_audit.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_audit.yml index a16bbf220ab..0bd6d609e19 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_audit.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_audit.yml @@ -2,356 +2,446 @@ description: Pipeline for Jamf Protect Telemetry audit logs. processors: - set: + tag: set_jamf_protect_telemetry_dataset_b7c3e9ce field: jamf_protect.telemetry.dataset value: audit - convert: + tag: convert_json__event_score_to_jamf_protect_telemetry_event_score_a944a062 field: json._event_score target_field: jamf_protect.telemetry.event_score type: long ignore_missing: true on_failure: - append: + tag: append_error_message_6e3c497b field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_header_event_id_to_event_code_ea6a00b9 field: json.header.event_id target_field: event.code type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_header_event_modifier_to_jamf_protect_telemetry_header_event_modifier_54f1efd8 field: json.header.event_modifier target_field: jamf_protect.telemetry.header.event_modifier type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_header_event_name_to_event_action_99289c09 field: json.header.event_name target_field: event.action ignore_missing: true - lowercase: + tag: lowercase_event_action_9334b869 field: event.action ignore_missing: true - script: + tag: script_ccf79a44 lang: painless if: ctx.json?.header?.time_seconds_epoch != 0 source: | ctx.json.time_milliseconds = (long)ctx.json?.header?.time_seconds_epoch * 1000 + (long)ctx.json?.header?.time_milliseconds_offset; - date: + tag: date_json_time_milliseconds_090aa438 field: json.time_milliseconds if: ctx.json?.time_milliseconds != 0 formats: - UNIX_MS on_failure: - append: + tag: append_error_message_1c8096c5 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_header_version_to_jamf_protect_telemetry_header_version_20dd329c field: json.header.version target_field: jamf_protect.telemetry.header.version type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_host_info_host_name_to_host_hostname_b155f2aa field: json.host_info.host_name target_field: host.hostname ignore_missing: true - append: + tag: append_related_hosts_e6462550 field: related.hosts value: '{{{host.hostname}}}' if: ctx.host?.hostname != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_host_info_host_uuid_to_jamf_protect_telemetry_host_info_host_uuid_b38624a6 field: json.host_info.host_uuid target_field: jamf_protect.telemetry.host_info.host.uuid ignore_missing: true - rename: + tag: rename_json_host_info_osversion_to_host_os_version_d3d10e97 field: json.host_info.osversion target_field: host.os.version ignore_missing: true - append: + tag: append_host_mac_ddef4d66 field: host.mac value: '{{{json.host_info.primary_mac_address}}}' if: ctx.json?.host_info?.primary_mac_address != null allow_duplicates: false ignore_failure: true - gsub: + tag: gsub_host_mac_d1faae6a field: host.mac pattern: '[-:.]' replacement: '-' ignore_missing: true - uppercase: + tag: uppercase_host_mac_73cd4413 field: host.mac ignore_missing: true - rename: + tag: rename_json_host_info_serial_number_to_host_id_02030592 field: json.host_info.serial_number target_field: host.id ignore_missing: true - rename: + tag: rename_json_return_description_to_jamf_protect_telemetry_return_description_48bccc5b field: json.return.description target_field: jamf_protect.telemetry.return.description ignore_missing: true - convert: + tag: convert_json_return_error_to_error_code_4de20ffe field: json.return.error target_field: error.code type: string ignore_missing: true ignore_failure: true - set: + tag: set_event_outcome_d9c276d7 field: event.outcome value: 'success' if: ctx.error?.code == '0' - set: + tag: set_event_outcome_6b78e138 field: event.outcome value: 'failure' if: ctx.error?.code != '0' - convert: + tag: convert_json_return_return_value_to_process_exit_code_a60d08a9 field: json.return.return_value target_field: process.exit_code type: long ignore_missing: true on_failure: - append: + tag: append_error_message_754f0640 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_subject_audit_id_to_process_real_user_id_25b75fe6 field: json.subject.audit_id target_field: process.real_user.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_subject_audit_user_name_to_process_real_user_name_9efe64b1 field: json.subject.audit_user_name target_field: process.real_user.name ignore_missing: true - append: + tag: append_related_user_54bf3eff field: related.user value: '{{{process.real_user.name}}}' if: ctx.process?.real_user?.name != null allow_duplicates: false ignore_failure: true - append: + tag: append_user_name_6d699a67 field: user.name value: '{{{json.subject.audit_user_name}}}' if: ctx.json?.subject?.audit_user_name != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_subject_effective_group_id_to_jamf_protect_telemetry_subject_effective_group_id_692351b4 field: json.subject.effective_group_id target_field: jamf_protect.telemetry.subject.effective.group.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_subject_effective_group_name_to_jamf_protect_telemetry_subject_effective_group_name_f9758471 field: json.subject.effective_group_name target_field: jamf_protect.telemetry.subject.effective.group.name ignore_missing: true - convert: + tag: convert_json_subject_effective_user_id_to_process_user_id_4f74864d field: json.subject.effective_user_id target_field: process.user.id type: string ignore_missing: true ignore_failure: true - append: + tag: append_user_name_6426d56f field: user.name value: '{{{json.subject.effective_user_name}}}' if: ctx.json?.subject?.effective_user_name != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_6dba669f field: related.user value: '{{{json.subject.effective_user_name}}}' if: ctx.json?.subject?.effective_user_name != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_subject_effective_user_name_to_process_user_name_5691197e field: json.subject.effective_user_name target_field: process.user.name ignore_missing: true - convert: + tag: convert_json_subject_group_id_to_process_real_group_id_ea90ca9c field: json.subject.group_id target_field: process.real_group.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_subject_group_name_to_process_real_group_name_81890bc5 field: json.subject.group_name target_field: process.real_group.name ignore_missing: true - rename: + tag: rename_json_subject_process_hash_to_process_hash_sha1_ec183692 field: json.subject.process_hash target_field: process.hash.sha1 ignore_missing: true - append: + tag: append_related_hash_8db6a16a field: related.hash value: '{{{process.hash.sha1}}}' if: ctx.process?.hash?.sha1 != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_subject_process_id_to_jamf_protect_telemetry_subject_process_pid_230f7d82 field: json.subject.process_id target_field: jamf_protect.telemetry.subject.process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_a52ed083 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_subject_process_name_to_jamf_protect_telemetry_subject_process_name_b517208c field: json.subject.process_name target_field: jamf_protect.telemetry.subject.process.name ignore_missing: true - convert: + tag: convert_json_subject_session_id_to_jamf_protect_telemetry_subject_session_id_f08fbd37 field: json.subject.session_id target_field: jamf_protect.telemetry.subject.session.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_subject_terminal_id_addr_to_jamf_protect_telemetry_subject_terminal_id_addr_d46a0b84 field: json.subject.terminal_id.addr target_field: jamf_protect.telemetry.subject.terminal_id.addr type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_subject_terminal_id_ip_address_9dc8b10a field: json.subject.terminal_id.ip_address type: ip ignore_missing: true on_failure: - remove: + tag: remove_json_subject_terminal_id_ip_address_d1da3917 field: json.subject.terminal_id.ip_address ignore_missing: true - append: + tag: append_error_message_4c3b7861 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_host_ip_e4f236dd field: host.ip value: '{{{json.subject.terminal_id.ip_address}}}' if: ctx.json?.subject?.terminal_id?.ip_address != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_ip_8541d760 field: related.ip value: '{{{json.subject.terminal_id.ip_address}}}' if: ctx.json?.subject?.terminal_id?.ip_address != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_subject_terminal_id_port_to_jamf_protect_telemetry_subject_terminal_id_port_17d35f61 field: json.subject.terminal_id.port target_field: jamf_protect.telemetry.subject.terminal_id.port type: long ignore_missing: true on_failure: - append: + tag: append_error_message_c1e07c52 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_subject_terminal_id_type_to_jamf_protect_telemetry_subject_terminal_id_type_1caf8fb2 field: json.subject.terminal_id.type target_field: jamf_protect.telemetry.subject.terminal_id.type type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_subject_user_id_to_user_id_19eb600e field: json.subject.user_id target_field: user.id type: string ignore_missing: true ignore_failure: true - append: + tag: append_user_name_11035ae3 field: user.name value: '{{{json.subject.user_name}}}' if: ctx.json?.subject?.user_name != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_acadb613 field: related.user value: '{{{json.subject.user_name}}}' if: ctx.json?.subject?.user_name != null allow_duplicates: false ignore_failure: true - append: + tag: append_event_type_8a66ccaa field: event.type value: info - set: + tag: set_event_kind_de80643c field: event.kind value: event - append: + tag: append_event_category_109b4d72 field: event.category value: authentication - pipeline: + tag: pipeline_c3719faa name: '{{ IngestPipeline "pipeline_aue_accept" }}' if: ctx.event?.action == 'aue_accept' - pipeline: + tag: pipeline_4591dc86 name: '{{ IngestPipeline "pipeline_aue_auth" }}' if: '["aue_auth_user", "aue_ssauthorize", "aue_ssauthmech"].contains(ctx.event?.action)' - pipeline: + tag: pipeline_310772b1 name: '{{ IngestPipeline "pipeline_aue_bind_and_aue_connect" }}' if: '["aue_bind", "aue_connect"].contains(ctx.event?.action)' - pipeline: + tag: pipeline_04eabdd8 name: '{{ IngestPipeline "pipeline_aue_chdir" }}' if: ctx.event?.action == 'aue_chdir' - pipeline: + tag: pipeline_dcf805ba name: '{{ IngestPipeline "pipeline_aue_chroot" }}' if: ctx.event?.action == 'aue_chroot' - pipeline: + tag: pipeline_69b2300a name: '{{ IngestPipeline "pipeline_aue_execve" }}' if: ctx.event?.action == 'aue_execve' - pipeline: + tag: pipeline_4ec40ca6 name: '{{ IngestPipeline "pipeline_aue_exit" }}' if: ctx.event?.action == 'aue_exit' - pipeline: + tag: pipeline_a6cfb1da name: '{{ IngestPipeline "pipeline_aue_kill" }}' if: ctx.event?.action == 'aue_kill' - pipeline: + tag: pipeline_16690266 name: '{{ IngestPipeline "pipeline_aue_mount" }}' if: ctx.event?.action == 'aue_mount' - pipeline: + tag: pipeline_40250022 name: '{{ IngestPipeline "pipeline_aue_posix_spawn" }}' if: ctx.event?.action == 'aue_posix_spawn' - pipeline: + tag: pipeline_100fa503 name: '{{ IngestPipeline "pipeline_aue_remove_from_group_and_aue_mac_set_proc" }}' if: '["aue_remove_from_group", "aue_mac_set_proc"].contains(ctx.event?.action)' - pipeline: + tag: pipeline_572cf3d4 name: '{{ IngestPipeline "pipeline_aue_session" }}' if: '["aue_session_end", "aue_session_update", "aue_session_close", "aue_session_start"].contains(ctx.event?.action)' - pipeline: + tag: pipeline_fb3fceb7 name: '{{ IngestPipeline "pipeline_aue_arguments" }}' if: '["aue_setsockopt", "aue_shutdown"].contains(ctx.event?.action)' - pipeline: + tag: pipeline_8f65aeba name: '{{ IngestPipeline "pipeline_aue_ssauthint" }}' if: ctx.event?.action == 'aue_ssauthint' - pipeline: + tag: pipeline_fa69ea72 name: '{{ IngestPipeline "pipeline_aue_tasknameforpid" }}' if: ctx.event?.action == 'aue_tasknameforpid' - pipeline: + tag: pipeline_180d23c4 name: '{{ IngestPipeline "pipeline_aue_unmount" }}' if: ctx.event?.action == 'aue_unmount' - pipeline: + tag: pipeline_010c0a1e name: '{{ IngestPipeline "pipeline_aue_fork" }}' if: ctx.event?.action == 'aue_fork' - pipeline: + tag: pipeline_04e114c6 name: '{{ IngestPipeline "pipeline_identity_object" }}' if: '["aue_getauid", "aue_lw_login", "aue_settimeofday"].contains(ctx.event?.action)' - pipeline: + tag: pipeline_d925ffca name: '{{ IngestPipeline "pipeline_aue_listen" }}' if: ctx.event?.action == 'aue_listen' - pipeline: + tag: pipeline_c67ebc32 name: '{{ IngestPipeline "pipeline_aue_logout" }}' if: ctx.event?.action == 'aue_logout' - pipeline: + tag: pipeline_e27168ba name: '{{ IngestPipeline "pipeline_aue_pidfortask" }}' if: ctx.event?.action == 'aue_pidfortask' - pipeline: + tag: pipeline_ecce8744 name: '{{ IngestPipeline "pipeline_aue_setpriority" }}' if: ctx.event?.action == 'aue_setpriority' - pipeline: + tag: pipeline_9f7644b2 name: '{{ IngestPipeline "pipeline_aue_socketpair" }}' if: ctx.event?.action == 'aue_socketpair' - pipeline: + tag: pipeline_577e20a2 name: '{{ IngestPipeline "pipeline_aue_taskforpid" }}' if: ctx.event?.action == 'aue_taskforpid' on_failure: - set: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_accept.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_accept.yml index 0ef1034e8bd..9d02d38d1c9 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_accept.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_accept.yml @@ -2,26 +2,31 @@ description: Pipeline for Jamf Protect Telemetry aue_accept audit logs. processors: - rename: + tag: rename_json_path_to_jamf_protect_telemetry_path_4323bfb1 field: json.path target_field: jamf_protect.telemetry.path ignore_missing: true - convert: + tag: convert_json_socket_unix_family_to_json_inet_family_a3c3ec3c field: json.socket_unix.family target_field: json.inet_family type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_socket_unix_path_to_jamf_protect_telemetry_socket_unix_path_2044a082 field: json.socket_unix.path target_field: jamf_protect.telemetry.socket.unix.path ignore_missing: true - convert: + tag: convert_json_arguments_fd_to_jamf_protect_telemetry_arguments_fd_72e625ba field: json.arguments.fd target_field: jamf_protect.telemetry.arguments.fd type: string ignore_missing: true ignore_failure: true - script: + tag: script_a4781b48 description: Dynamically map Socket Address Families. lang: painless if: ctx.json?.inet_family != null @@ -57,4 +62,5 @@ processors: map.put('25', 'pseudo_AF_PIP'); ctx.jamf_protect.telemetry.socket.unix.family = map.get(ctx.json.inet_family); - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_arguments.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_arguments.yml index 82c19b19a57..0eab230537e 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_arguments.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_arguments.yml @@ -2,10 +2,12 @@ description: Pipeline for Jamf Protect Telemetry aue_arguments audit logs. processors: - convert: + tag: convert_json_arguments_fd_to_jamf_protect_telemetry_arguments_fd_72e625ba field: json.arguments.fd target_field: jamf_protect.telemetry.arguments.fd type: string ignore_missing: true ignore_failure: true - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_auth.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_auth.yml index 5b6bbb7d842..bfbfbeab2dc 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_auth.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_auth.yml @@ -2,8 +2,10 @@ description: Pipeline for Jamf Protect Telemetry aue_auth audit logs. processors: - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' - rename: + tag: rename_json_texts_to_jamf_protect_telemetry_texts_a3c7f96b field: json.texts target_field: jamf_protect.telemetry.texts ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_bind_and_aue_connect.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_bind_and_aue_connect.yml index 3d3d4c5e2d6..8bff99aef38 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_bind_and_aue_connect.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_bind_and_aue_connect.yml @@ -2,57 +2,68 @@ description: Pipeline for Jamf Protect Telemetry aue_bind and aue_connect audit logs. processors: - convert: + tag: convert_json_socket_inet_addr_to_jamf_protect_telemetry_socket_inet_addr_cb71872f field: json.socket_inet.addr target_field: jamf_protect.telemetry.socket.inet.addr type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_arguments_fd_to_jamf_protect_telemetry_arguments_fd_72e625ba field: json.arguments.fd target_field: jamf_protect.telemetry.arguments.fd type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_socket_inet_family_to_json_inet_family_da3c0fb0 field: json.socket_inet.family target_field: json.inet_family type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_socket_inet_id_to_jamf_protect_telemetry_socket_inet_id_20112473 field: json.socket_inet.id target_field: jamf_protect.telemetry.socket.inet.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_socket_inet_ip_address_to_server_ip_b5e6ed72 field: json.socket_inet.ip_address target_field: server.ip type: ip ignore_missing: true on_failure: - remove: + tag: remove_json_socket_inet_ip_address_f04d008f field: json.socket_inet.ip_address ignore_missing: true - append: + tag: append_error_message_106415f7 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_related_ip_3ed00a18 field: related.ip value: '{{{server.ip}}}' if: ctx.server?.ip != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_socket_inet_port_to_server_port_91beb58f field: json.socket_inet.port target_field: server.port type: long ignore_missing: true on_failure: - append: + tag: append_error_message_6c7bb514 field: error.message value: '{{{_ingest.on_failure_message}}}' - script: + tag: script_d83587c4 description: Dynamically map Socket Address Families. lang: painless if: ctx.json?.inet_family != null @@ -88,4 +99,5 @@ processors: map.put('25', 'pseudo_AF_PIP'); ctx.jamf_protect.telemetry.socket.inet.family = map.get(ctx.json.inet_family); - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_chdir.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_chdir.yml index e2bc1de2814..8a06da31d5c 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_chdir.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_chdir.yml @@ -2,67 +2,80 @@ description: Pipeline for Jamf Protect Telemetry aue_chdir audit logs. processors: - convert: + tag: convert_json_attributes_device_to_jamf_protect_telemetry_attributes_device_5738d4f4 field: json.attributes.device target_field: jamf_protect.telemetry.attributes.device type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_attributes_file_access_mode_to_json_file_access_mode_3515c381 field: json.attributes.file_access_mode target_field: json.file_access_mode ignore_failure: true - convert: + tag: convert_json_attributes_file_system_id_to_jamf_protect_telemetry_attributes_file_system_id_5b5d2ea2 field: json.attributes.file_system_id target_field: jamf_protect.telemetry.attributes.file.system.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_attributes_node_id_to_jamf_protect_telemetry_attributes_node_id_78f34dc5 field: json.attributes.node_id target_field: jamf_protect.telemetry.attributes.node.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_attributes_owner_group_id_to_user_group_id_f8a03798 field: json.attributes.owner_group_id target_field: user.group.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_attributes_owner_group_name_to_user_group_name_834f090f field: json.attributes.owner_group_name target_field: user.group.name ignore_missing: true - convert: + tag: convert_json_attributes_owner_user_id_58bbc519 field: json.attributes.owner_user_id type: string ignore_missing: true ignore_failure: true - append: + tag: append_user_id_f10ea64d field: user.id value: '{{{json.attributes.owner_user_id}}}' if: ctx.json?.attributes?.owner_user_id != null allow_duplicates: false ignore_failure: true - append: + tag: append_user_name_2ba08467 field: user.name value: '{{{json.attributes.owner_user_name}}}' if: ctx.json?.attributes?.owner_user_name != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_0321d5d7 field: related.user value: '{{{json.attributes.owner_user_name}}}' if: ctx.json?.attributes?.owner_user_name != null allow_duplicates: false ignore_failure: true - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' - rename: + tag: rename_json_path_to_jamf_protect_telemetry_path_4323bfb1 field: json.path target_field: jamf_protect.telemetry.path ignore_missing: true - script: + tag: script_7f4421a4 description: Convert Decimal into Octal. lang: painless source: | diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_chroot.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_chroot.yml index d695d2efe4a..5edd81268b5 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_chroot.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_chroot.yml @@ -2,69 +2,83 @@ description: Pipeline for Jamf Protect Telemetry aue_chroot audit logs. processors: - convert: + tag: convert_json_attributes_device_to_jamf_protect_telemetry_attributes_device_5738d4f4 field: json.attributes.device target_field: jamf_protect.telemetry.attributes.device type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_attributes_file_access_mode_to_json_file_access_mode_58473c4f field: json.attributes.file_access_mode target_field: json.file_access_mode ignore_missing: true - convert: + tag: convert_json_attributes_file_system_id_to_jamf_protect_telemetry_attributes_file_system_id_5b5d2ea2 field: json.attributes.file_system_id target_field: jamf_protect.telemetry.attributes.file.system.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_attributes_node_id_to_jamf_protect_telemetry_attributes_node_id_78f34dc5 field: json.attributes.node_id target_field: jamf_protect.telemetry.attributes.node.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_attributes_owner_group_id_to_user_group_id_f8a03798 field: json.attributes.owner_group_id target_field: user.group.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_attributes_owner_group_name_to_user_group_name_834f090f field: json.attributes.owner_group_name target_field: user.group.name ignore_missing: true - convert: + tag: convert_json_attributes_owner_user_id_58bbc519 field: json.attributes.owner_user_id type: string ignore_missing: true ignore_failure: true - append: + tag: append_user_id_f10ea64d field: user.id value: '{{{json.attributes.owner_user_id}}}' if: ctx.json?.attributes?.owner_user_id != null allow_duplicates: false ignore_failure: true - append: + tag: append_user_name_2ba08467 field: user.name value: '{{{json.attributes.owner_user_name}}}' if: ctx.json?.attributes?.owner_user_name != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_0321d5d7 field: related.user value: '{{{json.attributes.owner_user_name}}}' if: ctx.json?.attributes?.owner_user_name != null allow_duplicates: false ignore_failure: true - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' - rename: + tag: rename_json_path_to_jamf_protect_telemetry_path_4323bfb1 field: json.path target_field: jamf_protect.telemetry.path ignore_missing: true - pipeline: + tag: pipeline_75de012d name: '{{ IngestPipeline "pipeline_exec_chain_child_object" }}' - script: + tag: script_7f4421a4 description: Convert Decimal into Octal. lang: painless source: | diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_execve.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_execve.yml index 663893d62a3..3239cbf7d4b 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_execve.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_execve.yml @@ -2,77 +2,93 @@ description: Pipeline for Jamf Protect Telemetry aue_execve audit logs. processors: - convert: + tag: convert_json_attributes_device_to_jamf_protect_telemetry_attributes_device_5738d4f4 field: json.attributes.device target_field: jamf_protect.telemetry.attributes.device type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_attributes_file_access_mode_to_json_file_access_mode_3515c381 field: json.attributes.file_access_mode target_field: json.file_access_mode ignore_failure: true - convert: + tag: convert_json_attributes_file_system_id_to_jamf_protect_telemetry_attributes_file_system_id_5b5d2ea2 field: json.attributes.file_system_id target_field: jamf_protect.telemetry.attributes.file.system.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_attributes_node_id_to_jamf_protect_telemetry_attributes_node_id_78f34dc5 field: json.attributes.node_id target_field: jamf_protect.telemetry.attributes.node.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_attributes_owner_group_id_to_user_group_id_f8a03798 field: json.attributes.owner_group_id target_field: user.group.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_attributes_owner_group_name_to_user_group_name_834f090f field: json.attributes.owner_group_name target_field: user.group.name ignore_missing: true - convert: + tag: convert_json_attributes_owner_user_id_58bbc519 field: json.attributes.owner_user_id type: string ignore_missing: true ignore_failure: true - append: + tag: append_user_id_f10ea64d field: user.id value: '{{{json.attributes.owner_user_id}}}' if: ctx.json?.attributes?.owner_user_id != null allow_duplicates: false ignore_failure: true - append: + tag: append_user_name_2ba08467 field: user.name value: '{{{json.attributes.owner_user_name}}}' if: ctx.json?.attributes?.owner_user_name != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_0321d5d7 field: related.user value: '{{{json.attributes.owner_user_name}}}' if: ctx.json?.attributes?.owner_user_name != null allow_duplicates: false ignore_failure: true - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' - rename: + tag: rename_json_path_to_jamf_protect_telemetry_path_4323bfb1 field: json.path target_field: jamf_protect.telemetry.path ignore_missing: true - pipeline: + tag: pipeline_75de012d name: '{{ IngestPipeline "pipeline_exec_chain_child_object" }}' - rename: + tag: rename_json_exec_args_args_to_json_args_c2752467 field: json.exec_args.args target_field: json.args ignore_missing: true - rename: + tag: rename_json_exec_args_args_compiled_to_jamf_protect_telemetry_exec_args_args_compiled_acb1ed1b field: json.exec_args.args_compiled target_field: jamf_protect.telemetry.exec_args.args_compiled ignore_missing: true - script: + tag: script_fdd491a2 lang: painless if: ctx.json?.exec_env?.env?.ARCH != null && ctx.json.exec_env.env.ARCH != '' params: @@ -105,26 +121,32 @@ processors: ctx.host.os.put('type', ctx.json.exec_env.env.ARCH); ctx.json.exec_env.env.remove('ARCH'); - rename: + tag: rename_json_exec_env_env_CPU_to_host_architecture_a6731bc7 field: json.exec_env.env.CPU target_field: host.architecture ignore_missing: true - rename: + tag: rename_json_exec_env_env_PATH_to_jamf_protect_telemetry_exec_env_env_path_368c684b field: json.exec_env.env.PATH target_field: jamf_protect.telemetry.exec_env.env.path ignore_missing: true - rename: + tag: rename_json_exec_env_env_XPC_FLAGS_to_jamf_protect_telemetry_exec_env_env_xpc_flags_57d43742 field: json.exec_env.env.XPC_FLAGS target_field: jamf_protect.telemetry.exec_env.env.xpc.flags ignore_missing: true - rename: + tag: rename_json_exec_env_env_XPC_SERVICE_NAME_to_jamf_protect_telemetry_exec_env_env_xpc_service_name_ac6aa5d2 field: json.exec_env.env.XPC_SERVICE_NAME target_field: jamf_protect.telemetry.exec_env.env.xpc.service_name ignore_missing: true - rename: + tag: rename_json_exec_env_env_compiled_to_jamf_protect_telemetry_exec_env_env_compiled_7ed739de field: json.exec_env.env_compiled target_field: jamf_protect.telemetry.exec_env.env.compiled ignore_missing: true - script: + tag: script_e6cd1664 description: Convert Object into Array. lang: painless source: | @@ -134,6 +156,7 @@ processors: ctx.process?.args.add(m.getValue()); } - script: + tag: script_7f4421a4 description: Convert Decimal into Octal. lang: painless source: | diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_exit.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_exit.yml index e16105fcf36..20ea90b6a80 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_exit.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_exit.yml @@ -2,19 +2,24 @@ description: Pipeline for Jamf Protect Telemetry aue_exit audit logs. processors: - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' - pipeline: + tag: pipeline_75de012d name: '{{ IngestPipeline "pipeline_exec_chain_child_object" }}' - convert: + tag: convert_json_exit_return_value_to_jamf_protect_telemetry_exit_return_value_3a3ebf30 field: json.exit.return_value target_field: jamf_protect.telemetry.exit.return.value type: long ignore_missing: true on_failure: - append: + tag: append_error_message_9a60d6c7 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_exit_status_to_jamf_protect_telemetry_exit_status_aa731c18 field: json.exit.status target_field: jamf_protect.telemetry.exit.status type: string diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_fork.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_fork.yml index 4bd0633dca5..1d49904d4a8 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_fork.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_fork.yml @@ -2,17 +2,21 @@ description: Pipeline for Jamf Protect Telemetry aue_fork audit logs. processors: - rename: + tag: rename_json_exec_chain_parent_uuid_to_jamf_protect_telemetry_exec_chain_parent_uuid_2d280d91 field: json.exec_chain_parent.uuid target_field: jamf_protect.telemetry.exec_chain_parent.uuid ignore_missing: true - convert: + tag: convert_json_arguments_child_PID_to_jamf_protect_telemetry_arguments_child_pid_c34b7804 field: json.arguments.child_PID target_field: jamf_protect.telemetry.arguments.child.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_cbc59557 field: error.message value: '{{{_ingest.on_failure_message}}}' - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_kill.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_kill.yml index 903b5f46a35..c3227dae1c7 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_kill.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_kill.yml @@ -2,12 +2,15 @@ description: Pipeline for Jamf Protect Telemetry aue_kill audit logs. processors: - convert: + tag: convert_json_arguments_signal_to_jamf_protect_telemetry_arguments_signal_e3188c02 field: json.arguments.signal target_field: jamf_protect.telemetry.arguments.signal type: string ignore_missing: true ignore_failure: true - pipeline: + tag: pipeline_b038445a name: '{{ IngestPipeline "pipeline_process_object" }}' - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_listen.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_listen.yml index 0d855e3fe9e..7121714aa48 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_listen.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_listen.yml @@ -2,12 +2,15 @@ description: Pipeline for Jamf Protect Telemetry aue_listen audit logs. processors: - convert: + tag: convert_json_arguments_fd_to_jamf_protect_telemetry_arguments_fd_72e625ba field: json.arguments.fd target_field: jamf_protect.telemetry.arguments.fd type: string ignore_missing: true ignore_failure: true - pipeline: + tag: pipeline_75de012d name: '{{ IngestPipeline "pipeline_exec_chain_child_object" }}' - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_logout.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_logout.yml index 9e83063d193..eb297b51686 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_logout.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_logout.yml @@ -2,6 +2,8 @@ description: Pipeline for Jamf Protect Telemetry aue_logout audit logs. processors: - pipeline: + tag: pipeline_75de012d name: '{{ IngestPipeline "pipeline_exec_chain_child_object" }}' - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_mount.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_mount.yml index 9fa589eca1e..6a161592f77 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_mount.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_mount.yml @@ -2,79 +2,95 @@ description: Pipeline for Jamf Protect Telemetry aue_mount audit logs. processors: - rename: + tag: rename_json_texts_to_jamf_protect_telemetry_texts_a3c7f96b field: json.texts target_field: jamf_protect.telemetry.texts ignore_missing: true - pipeline: + tag: pipeline_75de012d name: '{{ IngestPipeline "pipeline_exec_chain_child_object" }}' - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' - rename: + tag: rename_json_path_to_jamf_protect_telemetry_path_4323bfb1 field: json.path target_field: jamf_protect.telemetry.path ignore_missing: true - convert: + tag: convert_json_attributes_device_to_jamf_protect_telemetry_attributes_device_5738d4f4 field: json.attributes.device target_field: jamf_protect.telemetry.attributes.device type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_attributes_file_access_mode_to_json_file_access_mode_58473c4f field: json.attributes.file_access_mode target_field: json.file_access_mode ignore_missing: true - convert: + tag: convert_json_attributes_file_system_id_to_jamf_protect_telemetry_attributes_file_system_id_5b5d2ea2 field: json.attributes.file_system_id target_field: jamf_protect.telemetry.attributes.file.system.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_attributes_node_id_to_jamf_protect_telemetry_attributes_node_id_78f34dc5 field: json.attributes.node_id target_field: jamf_protect.telemetry.attributes.node.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_attributes_owner_group_id_to_jamf_protect_telemetry_attributes_owner_group_id_79477094 field: json.attributes.owner_group_id target_field: jamf_protect.telemetry.attributes.owner.group.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_attributes_owner_group_name_to_jamf_protect_telemetry_attributes_owner_group_name_9dba5003 field: json.attributes.owner_group_name target_field: jamf_protect.telemetry.attributes.owner.group.name ignore_missing: true - convert: + tag: convert_json_attributes_owner_user_id_58bbc519 field: json.attributes.owner_user_id type: string ignore_missing: true ignore_failure: true - append: + tag: append_user_id_f10ea64d field: user.id value: '{{{json.attributes.owner_user_id}}}' if: ctx.json?.attributes?.owner_user_id != null allow_duplicates: false ignore_failure: true - append: + tag: append_user_name_2ba08467 field: user.name value: '{{{json.attributes.owner_user_name}}}' if: ctx.json?.attributes?.owner_user_name != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_0321d5d7 field: related.user value: '{{{json.attributes.owner_user_name}}}' if: ctx.json?.attributes?.owner_user_name != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_arguments_flags_to_jamf_protect_telemetry_arguments_flags_f83dec40 field: json.arguments.flags target_field: jamf_protect.telemetry.arguments.flags type: string ignore_missing: true ignore_failure: true - script: + tag: script_7f4421a4 description: Convert Decimal into Octal. lang: painless source: | diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_pidfortask.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_pidfortask.yml index f9628191285..6c7207cfc10 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_pidfortask.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_pidfortask.yml @@ -2,20 +2,24 @@ description: Pipeline for Jamf Protect Telemetry aue_pidfortask audit logs. processors: - convert: + tag: convert_json_arguments_pid_to_jamf_protect_telemetry_arguments_pid_a3e34949 field: json.arguments.pid target_field: jamf_protect.telemetry.arguments.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_fe134aea field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_arguments_port_to_jamf_protect_telemetry_arguments_port_b699d42d field: json.arguments.port target_field: jamf_protect.telemetry.arguments.port type: long ignore_missing: true on_failure: - append: + tag: append_error_message_576e011e field: error.message value: '{{{_ingest.on_failure_message}}}' diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_posix_spawn.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_posix_spawn.yml index 1344c6fdb55..41d7a0fc8a7 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_posix_spawn.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_posix_spawn.yml @@ -2,39 +2,48 @@ description: Pipeline for Jamf Protect Telemetry aue_posix_spawn audit logs. processors: - convert: + tag: convert_json_arguments_child_PID_to_jamf_protect_telemetry_arguments_child_pid_c34b7804 field: json.arguments.child_PID target_field: jamf_protect.telemetry.arguments.child.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_cbc59557 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_exec_args_args_to_json_args_c2752467 field: json.exec_args.args target_field: json.args ignore_missing: true - rename: + tag: rename_json_exec_args_args_compiled_to_jamf_protect_telemetry_exec_args_args_compiled_acb1ed1b field: json.exec_args.args_compiled target_field: jamf_protect.telemetry.exec_args.args_compiled ignore_missing: true - rename: + tag: rename_json_exec_env_env_XPC_FLAGS_to_jamf_protect_telemetry_exec_env_env_xpc_flags_57d43742 field: json.exec_env.env.XPC_FLAGS target_field: jamf_protect.telemetry.exec_env.env.xpc.flags ignore_missing: true - rename: + tag: rename_json_exec_env_env_compiled_to_jamf_protect_telemetry_exec_env_env_compiled_7ed739de field: json.exec_env.env_compiled target_field: jamf_protect.telemetry.exec_env.env.compiled ignore_missing: true - rename: + tag: rename_json_path_to_jamf_protect_telemetry_path_4323bfb1 field: json.path target_field: jamf_protect.telemetry.path ignore_missing: true - rename: + tag: rename_json_exec_chain_parent_uuid_to_jamf_protect_telemetry_exec_chain_parent_uuid_2d280d91 field: json.exec_chain_parent.uuid target_field: jamf_protect.telemetry.exec_chain_parent.uuid ignore_missing: true - script: + tag: script_e6cd1664 description: Convert Object into Array. lang: painless source: | @@ -44,4 +53,5 @@ processors: ctx.process?.args.add(m.getValue()); } - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_remove_from_group_and_aue_mac_set_proc.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_remove_from_group_and_aue_mac_set_proc.yml index 95aaec8433a..a0a082a5e28 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_remove_from_group_and_aue_mac_set_proc.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_remove_from_group_and_aue_mac_set_proc.yml @@ -2,10 +2,13 @@ description: Pipeline for Jamf Protect Telemetry aue_remove_from_group and aue_mac_set_proc audit logs. processors: - rename: + tag: rename_json_texts_to_jamf_protect_telemetry_texts_a3c7f96b field: json.texts target_field: jamf_protect.telemetry.texts ignore_missing: true - pipeline: + tag: pipeline_75de012d name: '{{ IngestPipeline "pipeline_exec_chain_child_object" }}' - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_session.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_session.yml index 70737f672e3..044c73eb6ca 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_session.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_session.yml @@ -2,18 +2,21 @@ description: Pipeline for Jamf Protect Telemetry aue_session_start, aue_session_end, aue_session_update and aue_session_close audit logs. processors: - convert: + tag: convert_json_arguments_am_failure_to_jamf_protect_telemetry_arguments_am_failure_69eaa5bc field: json.arguments.am_failure target_field: jamf_protect.telemetry.arguments.am_failure type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_arguments_am_success_to_jamf_protect_telemetry_arguments_am_success_fa6404d2 field: json.arguments.am_success target_field: jamf_protect.telemetry.arguments.am_success type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_arguments_sflags_to_jamf_protect_telemetry_arguments_sflags_05846726 field: json.arguments.sflags target_field: jamf_protect.telemetry.arguments.sflags type: string diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_setpriority.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_setpriority.yml index 1eed014df7d..03b8be83739 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_setpriority.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_setpriority.yml @@ -2,23 +2,28 @@ description: Pipeline for Jamf Protect Telemetry aue_setpriority audit logs. processors: - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' - convert: + tag: convert_json_arguments_priority_to_jamf_protect_telemetry_arguments_priority_32190dc1 field: json.arguments.priority target_field: jamf_protect.telemetry.arguments.priority type: long ignore_missing: true on_failure: - append: + tag: append_error_message_1fe14122 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_arguments_which_to_jamf_protect_telemetry_arguments_which_be1e0fd4 field: json.arguments.which target_field: jamf_protect.telemetry.arguments.which type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_arguments_who_to_jamf_protect_telemetry_arguments_who_c7567df0 field: json.arguments.who target_field: jamf_protect.telemetry.arguments.who type: string diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_socketpair.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_socketpair.yml index 8b560e1def8..7892ada8821 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_socketpair.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_socketpair.yml @@ -2,20 +2,24 @@ description: Pipeline for Jamf Protect Telemetry aue_socketpair audit logs. processors: - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' - convert: + tag: convert_json_arguments_domain_to_jamf_protect_telemetry_arguments_domain_f80b9a96 field: json.arguments.domain target_field: jamf_protect.telemetry.arguments.domain type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_arguments_protocol_to_jamf_protect_telemetry_arguments_protocol_547bb1a6 field: json.arguments.protocol target_field: jamf_protect.telemetry.arguments.protocol type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_arguments_type_to_jamf_protect_telemetry_arguments_type_a3d19fe6 field: json.arguments.type target_field: jamf_protect.telemetry.arguments.type type: string diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_ssauthint.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_ssauthint.yml index 4c2ffa716da..0161e7addb5 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_ssauthint.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_ssauthint.yml @@ -2,18 +2,22 @@ description: Pipeline for Jamf Protect Telemetry aue_ssauthint audit logs. processors: - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' - rename: + tag: rename_json_texts_to_jamf_protect_telemetry_texts_a3c7f96b field: json.texts target_field: jamf_protect.telemetry.texts ignore_missing: true - convert: + tag: convert_json_arguments_known_UID__to_jamf_protect_telemetry_arguments_known_uid_d7e99e9b field: json.arguments.known_UID_ target_field: jamf_protect.telemetry.arguments.known_uid type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_arguments_to_jamf_protect_telemetry_arguments_flattened_ea764fd4 field: json.arguments target_field: jamf_protect.telemetry.arguments.flattened ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_taskforpid.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_taskforpid.yml index 636b41e7c7f..2a4bf139773 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_taskforpid.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_taskforpid.yml @@ -2,22 +2,27 @@ description: Pipeline for Jamf Protect Telemetry aue_taskforpid audit logs. processors: - convert: + tag: convert_json_arguments_target_port_to_jamf_protect_telemetry_arguments_target_port_59d3a12c field: json.arguments.target_port target_field: jamf_protect.telemetry.arguments.target.port type: long ignore_missing: true on_failure: - append: + tag: append_error_message_fdf2775b field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_arguments_task_port_to_jamf_protect_telemetry_arguments_task_port_0a3fff6c field: json.arguments.task_port target_field: jamf_protect.telemetry.arguments.task.port type: long ignore_missing: true on_failure: - append: + tag: append_error_message_7096f003 field: error.message value: '{{{_ingest.on_failure_message}}}' - pipeline: + tag: pipeline_b038445a name: '{{ IngestPipeline "pipeline_process_object" }}' diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_tasknameforpid.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_tasknameforpid.yml index 20a0756e242..a02df717a06 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_tasknameforpid.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_tasknameforpid.yml @@ -2,26 +2,31 @@ description: Pipeline for Jamf Protect Telemetry aue_tasknameforpid audit logs. processors: - convert: + tag: convert_json_arguments_process_to_jamf_protect_telemetry_arguments_process_0045b5ec field: json.arguments.process target_field: jamf_protect.telemetry.arguments.process type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_arguments_target_port_to_jamf_protect_telemetry_arguments_target_port_59d3a12c field: json.arguments.target_port target_field: jamf_protect.telemetry.arguments.target.port type: long ignore_missing: true on_failure: - append: + tag: append_error_message_fdf2775b field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_arguments_task_port_to_jamf_protect_telemetry_arguments_task_port_0a3fff6c field: json.arguments.task_port target_field: jamf_protect.telemetry.arguments.task.port type: long ignore_missing: true on_failure: - append: + tag: append_error_message_7096f003 field: error.message value: '{{{_ingest.on_failure_message}}}' diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_unmount.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_unmount.yml index e0b03bf2c0c..7f19ae88cfa 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_unmount.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_aue_unmount.yml @@ -2,69 +2,83 @@ description: Pipeline for Jamf Protect Telemetry aue_unmount audit logs. processors: - pipeline: + tag: pipeline_8cb224f5 name: '{{ IngestPipeline "pipeline_identity_object" }}' - pipeline: + tag: pipeline_75de012d name: '{{ IngestPipeline "pipeline_exec_chain_child_object" }}' - rename: + tag: rename_json_path_to_jamf_protect_telemetry_path_4323bfb1 field: json.path target_field: jamf_protect.telemetry.path ignore_missing: true - convert: + tag: convert_json_attributes_device_to_jamf_protect_telemetry_attributes_device_5738d4f4 field: json.attributes.device target_field: jamf_protect.telemetry.attributes.device type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_attributes_file_access_mode_to_json_file_access_mode_58473c4f field: json.attributes.file_access_mode target_field: json.file_access_mode ignore_missing: true - convert: + tag: convert_json_attributes_file_system_id_to_jamf_protect_telemetry_attributes_file_system_id_5b5d2ea2 field: json.attributes.file_system_id target_field: jamf_protect.telemetry.attributes.file.system.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_attributes_node_id_to_jamf_protect_telemetry_attributes_node_id_78f34dc5 field: json.attributes.node_id target_field: jamf_protect.telemetry.attributes.node.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_attributes_owner_group_id_to_jamf_protect_telemetry_attributes_owner_group_id_79477094 field: json.attributes.owner_group_id target_field: jamf_protect.telemetry.attributes.owner.group.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_attributes_owner_group_name_to_jamf_protect_telemetry_attributes_owner_group_name_9dba5003 field: json.attributes.owner_group_name target_field: jamf_protect.telemetry.attributes.owner.group.name ignore_missing: true - convert: + tag: convert_json_attributes_owner_user_id_58bbc519 field: json.attributes.owner_user_id type: string ignore_missing: true ignore_failure: true - append: + tag: append_user_id_f10ea64d field: user.id value: '{{{json.attributes.owner_user_id}}}' if: ctx.json?.attributes?.owner_user_id != null allow_duplicates: false ignore_failure: true - append: + tag: append_user_name_2ba08467 field: user.name value: '{{{json.attributes.owner_user_name}}}' if: ctx.json?.attributes?.owner_user_name != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_0321d5d7 field: related.user value: '{{{json.attributes.owner_user_name}}}' if: ctx.json?.attributes?.owner_user_name != null allow_duplicates: false ignore_failure: true - script: + tag: script_7f4421a4 description: Convert Decimal into Octal. lang: painless source: | diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_event.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_event.yml index 445ee14740a..2bf9f40ee34 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_event.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_event.yml @@ -2,78 +2,97 @@ description: Pipeline for Jamf Protect Telemetry event logs. processors: - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_jamf_protect_telemetry_dataset_dcc391db field: jamf_protect.telemetry.dataset value: event - append: + tag: append_event_category_8a525c81 field: event.category value: process - set: + tag: set_host_os_type_074dd92b field: host.os.type value: macos - append: + tag: append_event_type_8a66ccaa field: event.type value: info - convert: + tag: convert_json__event_score_to_jamf_protect_telemetry_event_score_a944a062 field: json._event_score target_field: jamf_protect.telemetry.event_score type: long ignore_missing: true on_failure: - append: + tag: append_error_message_6e3c497b field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_header_event_name_to_event_action_99289c09 field: json.header.event_name target_field: event.action ignore_missing: true - lowercase: + tag: lowercase_event_action_9334b869 field: event.action ignore_missing: true - date: + tag: date_json_header_time_seconds_epoch_726cfcbc field: json.header.time_seconds_epoch if: ctx.json?.header?.time_seconds_epoch != 0 formats: - UNIX on_failure: - append: + tag: append_error_message_3be7c1d1 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_host_info_host_name_to_host_hostname_b155f2aa field: json.host_info.host_name target_field: host.hostname ignore_missing: true - append: + tag: append_related_hosts_e6462550 field: related.hosts value: '{{{host.hostname}}}' if: ctx.host?.hostname != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_host_info_host_uuid_to_jamf_protect_telemetry_host_info_host_uuid_b38624a6 field: json.host_info.host_uuid target_field: jamf_protect.telemetry.host_info.host.uuid ignore_missing: true - rename: + tag: rename_json_host_info_osversion_to_host_os_version_d3d10e97 field: json.host_info.osversion target_field: host.os.version ignore_missing: true - append: + tag: append_host_mac_ddef4d66 field: host.mac value: '{{{json.host_info.primary_mac_address}}}' if: ctx.json?.host_info?.primary_mac_address != null allow_duplicates: false ignore_failure: true - gsub: + tag: gsub_host_mac_d1faae6a field: host.mac pattern: '[-:.]' replacement: '-' ignore_missing: true - uppercase: + tag: uppercase_host_mac_73cd4413 field: host.mac ignore_missing: true - rename: + tag: rename_json_host_info_serial_number_to_host_id_02030592 field: json.host_info.serial_number target_field: host.id ignore_missing: true @@ -81,3 +100,11 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_exec_chain_child_object.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_exec_chain_child_object.yml index 790f00422f9..3841d2c3458 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_exec_chain_child_object.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_exec_chain_child_object.yml @@ -2,19 +2,23 @@ description: Pipeline for Jamf Protect Telemetry exec_chain_child_object audit logs. processors: - rename: + tag: rename_json_exec_chain_child_parent_path_to_jamf_protect_telemetry_exec_chain_child_parent_path_27f4f142 field: json.exec_chain_child.parent_path target_field: jamf_protect.telemetry.exec_chain_child.parent.path ignore_missing: true - convert: + tag: convert_json_exec_chain_child_parent_pid_to_process_parent_pid_409cd750 field: json.exec_chain_child.parent_pid target_field: process.parent.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_ea707b8b field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_exec_chain_child_parent_uuid_to_jamf_protect_telemetry_exec_chain_child_parent_uuid_1354e58a field: json.exec_chain_child.parent_uuid target_field: jamf_protect.telemetry.exec_chain_child.parent.uuid ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_identity_object.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_identity_object.yml index 69170ac5549..dd43bd6525a 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_identity_object.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_identity_object.yml @@ -2,32 +2,38 @@ description: Pipeline for Jamf Protect Telemetry identity_object audit logs. processors: - rename: + tag: rename_json_identity_cd_hash_to_jamf_protect_telemetry_identity_cd_hash_a9366b3b field: json.identity.cd_hash target_field: jamf_protect.telemetry.identity.cd_hash ignore_missing: true - append: + tag: append_related_hash_1f8b0198 field: related.hash value: '{{{jamf_protect.telemetry.identity.cd_hash}}}' if: ctx.jamf_compliance_reporter?.log?.identity?.cd_hash != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_identity_signer_id_to_process_code_signature_signing_id_a1f12da8 field: json.identity.signer_id target_field: process.code_signature.signing_id ignore_missing: true - convert: + tag: convert_json_identity_signer_id_truncated_to_jamf_protect_telemetry_identity_signer_id_truncated_e8b490c5 field: json.identity.signer_id_truncated target_field: jamf_protect.telemetry.identity.signer.id_truncated type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_identity_signer_type_to_jamf_protect_telemetry_identity_signer_type_50bcc329 field: json.identity.signer_type target_field: jamf_protect.telemetry.identity.signer.type type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_identity_team_id_to_process_code_signature_team_id_1390fb3f field: json.identity.team_id target_field: process.code_signature.team_id ignore_missing: true diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_process_object.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_process_object.yml index 5c4571a5ed3..151f10ae4d2 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_process_object.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_process_object.yml @@ -2,143 +2,171 @@ description: Pipeline for Jamf process_object audit logs. processors: - convert: + tag: convert_json_process_audit_id_to_jamf_protect_telemetry_process_pid_5f663da1 field: json.process.audit_id target_field: jamf_protect.telemetry.process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_66da9c04 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_process_effective_group_id_to_jamf_protect_telemetry_process_effective_group_id_a5d9e8a2 field: json.process.effective_group_id target_field: jamf_protect.telemetry.process.effective.group.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_process_effective_group_name_to_jamf_protect_telemetry_process_effective_group_name_333c19bf field: json.process.effective_group_name target_field: jamf_protect.telemetry.process.effective.group.name ignore_missing: true - convert: + tag: convert_json_process_effective_user_id_to_jamf_protect_telemetry_process_effective_user_id_2fd9cc26 field: json.process.effective_user_id target_field: jamf_protect.telemetry.process.effective.user.id type: string ignore_missing: true ignore_failure: true - append: + tag: append_user_effective_id_79bcf892 field: user.effective.id value: '{{{json.process.effective_user_id}}}' if: ctx.json?.process?.effective_user_id != null allow_duplicates: false ignore_failure: true - append: + tag: append_user_effective_name_ad60ed3c field: user.effective.name value: '{{{json.process.effective_user_name}}}' if: ctx.json?.process?.effective_user_name != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_process_effective_user_name_to_jamf_protect_telemetry_process_effective_user_name_a68ab9d9 field: json.process.effective_user_name target_field: jamf_protect.telemetry.process.effective.user.name ignore_missing: true - append: + tag: append_related_user_4235b529 field: related.user value: '{{{jamf_protect.telemetry.process.effective.user.name}}}' if: ctx.jamf_compliance_reporter?.log?.process?.effective?.user?.name != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_process_group_id_to_jamf_protect_telemetry_process_group_id_f40b4c73 field: json.process.group_id target_field: jamf_protect.telemetry.process.group.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_process_group_name_to_jamf_protect_telemetry_process_group_name_31ce04da field: json.process.group_name target_field: jamf_protect.telemetry.process.group.name ignore_missing: true - append: + tag: append_process_hash_sha1_0bcfc307 field: process.hash.sha1 value: '{{{json.process.process_hash}}}' if: ctx.json?.process?.process_hash != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_hash_18e9b98c field: related.hash value: '{{{json.process.process_hash}}}' if: ctx.json?.process?.process_hash != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_process_process_id_to_process_pid_01d6e1c0 field: json.process.process_id target_field: process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_65c342d3 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_process_process_name_to_process_name_19b5c16c field: json.process.process_name target_field: process.name ignore_missing: true - convert: + tag: convert_json_process_session_id_to_jamf_protect_telemetry_process_session_id_70e3409d field: json.process.session_id target_field: jamf_protect.telemetry.process.session.id type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_process_terminal_id_addr_to_jamf_protect_telemetry_process_terminal_id_addr_77468a5a field: json.process.terminal_id.addr target_field: jamf_protect.telemetry.process.terminal_id.addr type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_process_terminal_id_ip_address_to_jamf_protect_telemetry_process_terminal_id_ip_address_5b9907b0 field: json.process.terminal_id.ip_address target_field: jamf_protect.telemetry.process.terminal_id.ip_address type: ip ignore_missing: true on_failure: - remove: + tag: remove_json_process_terminal_id_ip_address_b85b39be field: json.process.terminal_id.ip_address ignore_missing: true - append: + tag: append_error_message_b40f3599 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_related_ip_8a311e0c field: related.ip value: '{{{jamf_protect.telemetry.process.terminal_id.ip_address}}}' if: ctx.jamf_compliance_reporter?.log?.process?.terminal_id?.ip_address != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_process_terminal_id_port_to_jamf_protect_telemetry_process_terminal_id_port_bfaaccd9 field: json.process.terminal_id.port target_field: jamf_protect.telemetry.process.terminal_id.port type: long ignore_missing: true on_failure: - append: + tag: append_error_message_986ddde2 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_process_terminal_id_type_to_jamf_protect_telemetry_process_terminal_id_type_26de5764 field: json.process.terminal_id.type target_field: jamf_protect.telemetry.process.terminal_id.type type: string ignore_missing: true ignore_failure: true - convert: + tag: convert_json_process_user_id_to_process_user_id_91823b46 field: json.process.user_id target_field: process.user.id type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_process_user_name_to_jamf_protect_telemetry_process_user_name_c800aaea field: json.process.user_name target_field: jamf_protect.telemetry.process.user.name ignore_missing: true - append: + tag: append_related_user_f7825426 field: related.user value: '{{{jamf_protect.telemetry.process.user.name}}}' if: ctx.jamf_compliance_reporter?.log?.process?.user?.name != null diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_system_performance_metrics.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_system_performance_metrics.yml index 324062e4dc5..ef8f8e72239 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_system_performance_metrics.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/pipeline_system_performance_metrics.yml @@ -2,136 +2,168 @@ description: Pipeline for Jamf Protect Telemetry app metrics logs. processors: - append: + tag: append_event_type_8a66ccaa field: event.type value: info - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_jamf_protect_telemetry_dataset_6f595180 field: jamf_protect.telemetry.dataset value: app_metrics - set: + tag: set_host_os_type_074dd92b field: host.os.type value: macos - append: + tag: append_event_category_8a525c81 field: event.category value: process - convert: + tag: convert_json__event_score_to_jamf_protect_telemetry_event_score_a944a062 field: json._event_score type: long target_field: jamf_protect.telemetry.event_score ignore_missing: true on_failure: - append: + tag: append_error_message_6e3c497b field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_system_performance_metrics_cpu_percentage_to_jamf_protect_telemetry_system_performance_metrics_cpu_percentage_66bc8100 field: json.system_performance_metrics.cpu_percentage type: double target_field: jamf_protect.telemetry.system_performance_metrics.cpu_percentage ignore_missing: true on_failure: - append: + tag: append_error_message_cbaca0fb field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_system_performance_metrics_cpu_time_seconds_to_jamf_protect_telemetry_system_performance_metrics_cpu_time_seconds_a4ee687a field: json.system_performance_metrics.cpu_time_seconds type: double target_field: jamf_protect.telemetry.system_performance_metrics.cpu_time_seconds ignore_missing: true on_failure: - append: + tag: append_error_message_900c1559 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_system_performance_metrics_interrupt_wakeups_to_jamf_protect_telemetry_system_performance_metrics_interrupt_wakeups_4581f915 field: json.system_performance_metrics.interrupt_wakeups type: long target_field: jamf_protect.telemetry.system_performance_metrics.interrupt_wakeups ignore_missing: true on_failure: - append: + tag: append_error_message_1246971e field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_system_performance_metrics_platform_idle_wakeups_to_jamf_protect_telemetry_system_performance_metrics_platform_idle_wakeups_403e5a91 field: json.system_performance_metrics.platform_idle_wakeups type: long target_field: jamf_protect.telemetry.system_performance_metrics.platform_idle_wakeups ignore_missing: true on_failure: - append: + tag: append_error_message_92c674b2 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_system_performance_metrics_resident_memory_size_mb_to_jamf_protect_telemetry_system_performance_metrics_resident_memory_size_mb_758f356d field: json.system_performance_metrics.resident_memory_size_mb type: double target_field: jamf_protect.telemetry.system_performance_metrics.resident_memory_size.mb ignore_missing: true on_failure: - append: + tag: append_error_message_721de4a2 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_system_performance_metrics_virtual_memory_size_mb_to_jamf_protect_telemetry_system_performance_metrics_virtual_memory_size_mb_5598ff89 field: json.system_performance_metrics.virtual_memory_size_mb type: double target_field: jamf_protect.telemetry.system_performance_metrics.virtual_memory_size.mb ignore_missing: true on_failure: - append: + tag: append_error_message_e80b39e2 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_header_event_name_to_event_action_99289c09 field: json.header.event_name target_field: event.action ignore_missing: true - lowercase: + tag: lowercase_event_action_9334b869 field: event.action ignore_missing: true - date: + tag: date_json_header_time_seconds_epoch_726cfcbc field: json.header.time_seconds_epoch if: ctx.json?.header?.time_seconds_epoch != 0 formats: - UNIX on_failure: - append: + tag: append_error_message_3be7c1d1 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_host_info_host_name_to_host_hostname_b155f2aa field: json.host_info.host_name target_field: host.hostname ignore_missing: true - append: + tag: append_related_hosts_e6462550 field: related.hosts value: '{{{host.hostname}}}' if: ctx.host?.hostname != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_host_info_host_uuid_to_jamf_protect_telemetry_host_info_host_uuid_b38624a6 field: json.host_info.host_uuid target_field: jamf_protect.telemetry.host_info.host.uuid ignore_missing: true - rename: + tag: rename_json_host_info_osversion_to_host_os_version_d3d10e97 field: json.host_info.osversion target_field: host.os.version ignore_missing: true - append: + tag: append_host_mac_ddef4d66 field: host.mac value: '{{{json.host_info.primary_mac_address}}}' if: ctx.json?.host_info?.primary_mac_address != null allow_duplicates: false ignore_failure: true - gsub: + tag: gsub_host_mac_d1faae6a field: host.mac pattern: '[-:.]' replacement: '-' ignore_missing: true - uppercase: + tag: uppercase_host_mac_73cd4413 field: host.mac ignore_missing: true - rename: + tag: rename_json_host_info_serial_number_to_host_id_02030592 field: json.host_info.serial_number target_field: host.id ignore_missing: true - script: + tag: script_68d4d1d0 lang: painless if: ctx.json?.system_performance_metrics?.cpu_percentage != null source: | @@ -141,3 +173,11 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/jamf_protect/data_stream/web_threat_events/elasticsearch/ingest_pipeline/default.yml b/packages/jamf_protect/data_stream/web_threat_events/elasticsearch/ingest_pipeline/default.yml index 9132528adb5..247b8569425 100644 --- a/packages/jamf_protect/data_stream/web_threat_events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/jamf_protect/data_stream/web_threat_events/elasticsearch/ingest_pipeline/default.yml @@ -2,225 +2,255 @@ description: Pipeline for parsing Jamf Protect Web Protection threat logs processors: - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - json: + tag: json_message_to_jamf_protect_threat_19580f82 field: message target_field: jamf_protect.threat if: ctx.message != null - rename: + tag: rename_json_to_jamf_protect_threat_d4d31d44 field: json target_field: jamf_protect.threat if: ctx.json != null -########################## -## ECS Observer Mapping ## -########################## + ########################## + ## ECS Observer Mapping ## + ########################## - set: + tag: set_observer_product_e40a396c field: observer.product value: Jamf Protect - set: + tag: set_observer_vendor_6a522e0a field: observer.vendor value: Jamf - set: + tag: set_observer_type_ae54f4bd field: observer.type value: Endpoint Security -####################### -## ECS Event Mapping ## -####################### + ####################### + ## ECS Event Mapping ## + ####################### - set: + tag: set_event_kind_39295792 field: event.kind value: alert - set: + tag: set_event_provider_7e881320 field: event.provider value: Jamf Protect - rename: + tag: rename_jamf_protect_threat_event_timestamp_to_event_start_2c89cb7d field: jamf_protect.threat.event.timestamp target_field: event.start if: ctx.jamf_protect?.threat?.event?.timestamp != null - append: + tag: append_event_category_3fa422a0 field: event.category value: host - append: + tag: append_event_category_8998f81d field: event.category value: malware if: ctx.jamf_protect?.threat?.event?.eventType?.name == 'MALWARE' - rename: + tag: rename_jamf_protect_threat_event_action_to_event_action_c9e473f8 field: jamf_protect.threat.event.action target_field: event.action ignore_missing: true if: ctx.jamf_protect?.threat?.event?.action != null - set: + tag: set_event_reason_624ddbe7 field: event.reason value: '{{{jamf_protect.threat.event.eventType.description}}}' if: ctx.jamf_protect?.threat?.event?.eventType?.description != null # ignore_missing: true - rename: + tag: rename_jamf_protect_threat_event_alertId_to_event_id_0eac0a93 field: jamf_protect.threat.event.alertId target_field: event.id ignore_missing: true if: ctx.jamf_protect?.threat?.event?.alertId != null - rename: + tag: rename_jamf_protect_threat_event_severity_to_event_severity_7248c525 field: jamf_protect.threat.event.severity target_field: event.severity ignore_missing: true if: ctx.jamf_protect?.threat?.event?.severity != null - rename: + tag: rename_jamf_protect_threat_event_eventUrl_to_event_url_8684d0bf field: jamf_protect.threat.event.eventUrl target_field: event.url ignore_missing: true if: ctx.jamf_protect?.threat?.event?.eventUrl != null - -####################### -## ECS Rule Mapping ## -####################### + ####################### + ## ECS Rule Mapping ## + ####################### - rename: + tag: rename_jamf_protect_threat_event_eventType_name_to_rule_name_810b38da field: jamf_protect.threat.event.eventType.name - target_field: rule.name + target_field: rule.name if: ctx.jamf_protect?.threat?.event?.eventType?.name != null ignore_missing: true - set: + tag: set_rule_description_f0c0edcb field: rule.description value: '{{{jamf_protect.threat.event.eventType.description}}}' if: ctx.jamf_protect?.threat?.event?.eventType?.description != null # ignore_missing: true - rename: + tag: rename_jamf_protect_threat_event_eventType_id_toString()_to_rule_id_cc36fe1d field: jamf_protect.threat.event.eventType.id.toString() target_field: rule.id if: ctx.jamf_protect?.threat?.event?.eventType?.id != null ignore_missing: true - -####################### -## ECS Host Mapping ## -####################### + ####################### + ## ECS Host Mapping ## + ####################### - rename: + tag: rename_jamf_protect_threat_event_device_userDeviceName_to_host_hostname_99d60132 field: jamf_protect.threat.event.device.userDeviceName target_field: host.hostname ignore_missing: true if: ctx.jamf_protect?.threat?.event?.device?.userDeviceName != null - rename: + tag: rename_jamf_protect_threat_event_device_deviceName_to_host_hostname_3651fc92 field: jamf_protect.threat.event.device.deviceName target_field: host.hostname ignore_missing: true if: ctx.jamf_protect?.threat?.event?.device?.deviceName == null && ctx.jamf_protect?.threat?.event?.device?.userDeviceName != null - rename: + tag: rename_jamf_protect_threat_event_device_deviceId_to_host_id_18fe9b70 field: jamf_protect.threat.event.device.deviceId target_field: host.id ignore_missing: true if: ctx.jamf_protect?.threat?.event?.device?.deviceId != null - rename: + tag: rename_jamf_protect_threat_event_device_os_to_host_os_full_0b13d038 field: jamf_protect.threat.event.device.os target_field: host.os.full if: ctx.jamf_protect?.threat?.event?.device?.os != null ignore_missing: true -######################## -## ECS User Mapping ## -######################## + ######################## + ## ECS User Mapping ## + ######################## - rename: + tag: rename_jamf_protect_threat_event_user_name_to_user_name_013ef2ce field: jamf_protect.threat.event.user.name target_field: user.name - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.threat?.event?.user?.name != null - rename: + tag: rename_jamf_protect_threat_event_user_email_to_user_email_2f09046d field: jamf_protect.threat.event.user.email target_field: user.email - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.threat?.event?.user?.email != null -######################## -## ECS GEO Mapping ## -######################## + ######################## + ## ECS GEO Mapping ## + ######################## - rename: + tag: rename_jamf_protect_threat_event_location_to_host_geo_country_iso_code_17606000 field: jamf_protect.threat.event.location target_field: host.geo.country_iso_code - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.threat?.event?.location != null - -####################### -## ECS Destination Mapping ## -####################### + ####################### + ## ECS Destination Mapping ## + ####################### - rename: + tag: rename_jamf_protect_threat_event_destination_ip_to_destination_address_e8e49248 field: jamf_protect.threat.event.destination.ip target_field: destination.address - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.threat?.event?.destination?.ip != null - rename: + tag: rename_jamf_protect_threat_event_destination_name_to_destination_domain_2a2eac9c field: jamf_protect.threat.event.destination.name target_field: destination.domain - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.threat?.event?.destination?.name != null - rename: + tag: rename_jamf_protect_threat_event_destination_port_to_destination_port_150ac799 field: jamf_protect.threat.event.destination.port target_field: destination.port - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.threat?.event?.destination?.port != null - -####################### -## ECS File Mapping ## -####################### + ####################### + ## ECS File Mapping ## + ####################### - rename: + tag: rename_jamf_protect_threat_event_app_name_to_file_name_dbbca645 field: jamf_protect.threat.event.app.name target_field: file.name - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.threat?.event?.app?.name != null - rename: + tag: rename_jamf_protect_threat_event_app_sha1_to_file_hash_sha1_243dc881 field: jamf_protect.threat.event.app.sha1 target_field: file.hash.sha1 - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.threat?.event?.app?.sha1 != null - rename: + tag: rename_jamf_protect_threat_event_app_sha256_to_file_hash_sha256_c9c97747 field: jamf_protect.threat.event.app.sha256 target_field: file.hash.sha256 - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.threat?.event?.app?.sha256 != null -####################### -## ECS Source Mapping ## -####################### + ####################### + ## ECS Source Mapping ## + ####################### - rename: + tag: rename_jamf_protect_threat_event_source_ip_to_source_address_2678a1b8 field: jamf_protect.threat.event.source.ip target_field: source.address - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.threat?.event?.destination?.ip != null - rename: + tag: rename_jamf_protect_threat_event_source_port_to_source_port_6d2a1cdc field: jamf_protect.threat.event.source.port target_field: source.port - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.threat?.event?.source?.port != null -# ######################## -# # ECS Organization Mapping ## -# ######################## + # ######################## + # # ECS Organization Mapping ## + # ######################## - rename: + tag: rename_jamf_protect_threat_event_account_customerId_to_organization_id_e08585f3 field: jamf_protect.threat.event.account.customerId target_field: organization.id ignore_missing: true if: ctx.jamf_protect?.threat?.event?.account?.customerId != null + ######################## + # ECS Related Mapping ## + ######################## - -######################## -# ECS Related Mapping ## -######################## - - -############# -## Cleanup ## -############# + ############# + ## Cleanup ## + ############# - remove: + tag: remove_924b382b field: - jamf_protect.threat - jamf_protect - message ignore_missing: true - script: + tag: script_258616c7 description: Drops null/empty values recursively. lang: painless source: | @@ -247,4 +277,4 @@ on_failure: allow_duplicates: false - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' \ No newline at end of file + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/jamf_protect/data_stream/web_traffic_events/elasticsearch/ingest_pipeline/default.yml b/packages/jamf_protect/data_stream/web_traffic_events/elasticsearch/ingest_pipeline/default.yml index d5533a1c40b..029de2dd736 100644 --- a/packages/jamf_protect/data_stream/web_traffic_events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/jamf_protect/data_stream/web_traffic_events/elasticsearch/ingest_pipeline/default.yml @@ -2,104 +2,124 @@ description: Pipeline for parsing Jamf Protect Web Protection traffic logs processors: - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - json: + tag: json_message_to_jamf_protect_traffic_e825ce31 field: message target_field: jamf_protect.traffic if: ctx.message != null - rename: + tag: rename_json_to_jamf_protect_traffic_767af61b field: json target_field: jamf_protect.traffic if: ctx.json != null -########################## -## ECS Observer Mapping ## -########################## + ########################## + ## ECS Observer Mapping ## + ########################## - set: + tag: set_observer_product_e40a396c field: observer.product value: Jamf Protect - set: + tag: set_observer_vendor_6a522e0a field: observer.vendor value: Jamf - set: + tag: set_observer_type_ae54f4bd field: observer.type value: Endpoint Security -####################### -## ECS Event Mapping ## -####################### + ####################### + ## ECS Event Mapping ## + ####################### - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_provider_7e881320 field: event.provider value: Jamf Protect - rename: + tag: rename_jamf_protect_traffic_event_timestamp_to_event_start_1cdc9dc5 field: jamf_protect.traffic.event.timestamp target_field: event.start if: ctx.jamf_protect?.traffic?.event?.timestamp != null - append: + tag: append_event_category_3fa422a0 field: event.category value: host - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_category_044a2c00 field: event.category value: malware if: ctx.jamf_protect?.traffic?.event?.threat?.types == 'malware' - set: + tag: set_event_action_50a2afa1 field: event.action value: '{{{jamf_protect.traffic.event.signatureId.name}}}' if: ctx.jamf_protect?.traffic?.event?.signatureId?.name != null - set: + tag: set_event_reason_8ef4c98b field: event.reason value: '{{{jamf_protect.traffic.event.threat.result}}}' if: ctx.jamf_protect?.traffic?.event?.threat?.result != null - append: + tag: append_event_outcome_5e0b9f8a field: event.outcome value: failure if: ctx.jamf_protect?.traffic?.event?.blocked == true - append: + tag: append_event_outcome_911c0f36 field: event.outcome value: success if: ctx.jamf_protect?.traffic?.event?.blocked == false # ignore_missing: true - -####################### -## ECS Rule Mapping ## -####################### + ####################### + ## ECS Rule Mapping ## + ####################### - set: + tag: set_rule_name_bc4ad9a8 field: rule.name value: '{{{jamf_protect.traffic.event.signatureId.name}}}' if: ctx.jamf_protect?.traffic?.event?.signatureId?.name != null # ignore_missing: true - rename: + tag: rename_jamf_protect_traffic_event_signatureId_id_toString()_to_rule_id_78fc9ed7 field: jamf_protect.traffic.event.signatureId.id.toString() target_field: rule.id if: ctx.jamf_protect?.traffic?.event?.signatureId?.id != null ignore_missing: true - -####################### -## ECS Host Mapping ## -####################### + ####################### + ## ECS Host Mapping ## + ####################### - rename: + tag: rename_jamf_protect_traffic_event_device_userDeviceName_to_host_hostname_0fb2d2d6 field: jamf_protect.traffic.event.device.userDeviceName target_field: host.hostname ignore_missing: true if: ctx.jamf_protect?.traffic?.event?.device?.userDeviceName != null - rename: + tag: rename_jamf_protect_traffic_event_device_deviceName_to_host_hostname_35c5f85f field: jamf_protect.traffic.event.device.deviceName target_field: host.hostname ignore_missing: true if: ctx.jamf_protect?.traffic?.event?.device?.deviceName == null && ctx.jamf_protect?.traffic?.event?.device?.userDeviceName != null - rename: + tag: rename_jamf_protect_traffic_event_device_deviceId_to_host_id_64cf30e6 field: jamf_protect.traffic.event.device.deviceId target_field: host.id ignore_missing: true @@ -110,140 +130,156 @@ processors: # ignore_missing: true # if: ctx.jamf_protect?.traffic?.event?.host?.ips != null && ctx.jamf_protect?.traffic?.event?.host?.ips != "" - rename: + tag: rename_jamf_protect_traffic_event_device_os_to_host_os_full_69569b90 field: jamf_protect.traffic.event.device.os target_field: host.os.full if: ctx.jamf_protect?.traffic?.event?.device?.os != null ignore_missing: true - append: + tag: append_host_os_type_856fcde5 field: host.os.type value: ios if: ctx.jamf_protect?.traffic?.event?.device?.osType == "IOS" - append: + tag: append_host_os_type_9b9e494c field: host.os.type value: macos if: ctx.jamf_protect?.traffic?.event?.device?.osType == "MAC_OS" -######################## -## ECS User Mapping ## -######################## - + ######################## + ## ECS User Mapping ## + ######################## - rename: + tag: rename_jamf_protect_traffic_event_user_name_to_user_name_f28c8f1e field: jamf_protect.traffic.event.user.name target_field: user.name - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.traffic?.event?.user?.name != null - rename: + tag: rename_jamf_protect_traffic_event_user_email_to_user_email_bbbeaccd field: jamf_protect.traffic.event.user.email target_field: user.email - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.traffic?.event?.user?.email != null - -######################## -## ECS DNS Mapping ## -######################## + ######################## + ## ECS DNS Mapping ## + ######################## - rename: + tag: rename_jamf_protect_traffic_event_tld_to_dns_question_top_level_domain_7f5d9d7f field: jamf_protect.traffic.event.tld target_field: dns.question.top_level_domain - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.traffic?.event?.tld != null - rename: + tag: rename_jamf_protect_traffic_event_domain_to_dns_question_registered_domain_5872b015 field: jamf_protect.traffic.event.domain target_field: dns.question.registered_domain - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.traffic?.event?.domain != null - rename: + tag: rename_jamf_protect_traffic_event_hostName_to_dns_question_name_cdb90a33 field: jamf_protect.traffic.event.hostName target_field: dns.question.name - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.traffic?.event?.hostName != null - - rename: + tag: rename_jamf_protect_traffic_event_dns_responseStatus_to_dns_response_code_b8ec27ba field: jamf_protect.traffic.event.dns.responseStatus target_field: dns.response_code - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.traffic?.event?.dns?.responseStatus != null - rename: + tag: rename_jamf_protect_traffic_event_dns_recordType_to_dns_answers_type_99addc3a field: jamf_protect.traffic.event.dns.recordType target_field: dns.answers.type - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.traffic?.event?.dns?.recordType != null - rename: + tag: rename_jamf_protect_traffic_event_dns_ttl_to_dns_answers_ttl_7c4887d8 field: jamf_protect.traffic.event.dns.ttl target_field: dns.answers.ttl - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.traffic?.event?.dns?.ttl != null - set: + tag: set_dns_resolved_ip_a329cffe field: dns.resolved_ip value: '{{{jamf_protect.traffic.event.destination.ip}}}' - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.traffic?.event?.destination?.ip != null -######################## -## ECS GEO Mapping ## -######################## + ######################## + ## ECS GEO Mapping ## + ######################## - rename: + tag: rename_jamf_protect_traffic_event_location_to_host_geo_country_iso_code_5022d5d0 field: jamf_protect.traffic.event.location target_field: host.geo.country_iso_code - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.traffic?.event?.location != null - -####################### -## ECS Destination Mapping ## -####################### + ####################### + ## ECS Destination Mapping ## + ####################### - rename: + tag: rename_jamf_protect_traffic_event_destination_ip_to_destination_address_1351f4c8 field: jamf_protect.traffic.event.destination.ip target_field: destination.address - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.traffic?.event?.destination?.ip != null - rename: + tag: rename_jamf_protect_traffic_event_destination_name_to_destination_domain_3789860c field: jamf_protect.traffic.event.destination.name target_field: destination.domain - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.traffic?.event?.destination?.name != null - rename: + tag: rename_jamf_protect_traffic_event_destination_port_to_destination_port_90e09379 field: jamf_protect.traffic.event.destination.port target_field: destination.port - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.traffic?.event?.destination?.port != null - -####################### -## ECS Source Mapping ## -####################### + ####################### + ## ECS Source Mapping ## + ####################### - rename: + tag: rename_jamf_protect_traffic_event_source_ip_to_source_address_0bab4910 field: jamf_protect.traffic.event.source.ip target_field: source.address - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.traffic?.event?.destination?.ip != null - rename: + tag: rename_jamf_protect_traffic_event_source_port_to_source_port_a64c44a4 field: jamf_protect.traffic.event.source.port target_field: source.port - # ignore_missing: true + # ignore_missing: true if: ctx.jamf_protect?.traffic?.event?.source?.port != null -# ######################## -# # ECS Organization Mapping ## -# ######################## + # ######################## + # # ECS Organization Mapping ## + # ######################## - rename: + tag: rename_jamf_protect_traffic_event_account_customerId_to_organization_id_3a9bde01 field: jamf_protect.traffic.event.account.customerId target_field: organization.id ignore_missing: true if: ctx.jamf_protect?.traffic?.event?.account?.customerId != null -############# -## Cleanup ## -############# + ############# + ## Cleanup ## + ############# - remove: + tag: remove_d5c8f784 field: - jamf_protect.traffic - jamf_protect - message ignore_missing: true - script: + tag: script_258616c7 description: Drops null/empty values recursively. lang: painless source: | @@ -271,4 +307,4 @@ on_failure: allow_duplicates: false - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' \ No newline at end of file + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/jamf_protect/manifest.yml b/packages/jamf_protect/manifest.yml index ecac8f16737..0fd55e1abf5 100644 --- a/packages/jamf_protect/manifest.yml +++ b/packages/jamf_protect/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.3 name: jamf_protect title: Jamf Protect -version: "3.4.0" +version: "3.4.1" description: Receives events from Jamf Protect with Elastic Agent. type: integration categories: diff --git a/packages/jumpcloud/changelog.yml b/packages/jumpcloud/changelog.yml index 9ac95c31160..e1c419de1fd 100644 --- a/packages/jumpcloud/changelog.yml +++ b/packages/jumpcloud/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.22.3" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.22.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/jumpcloud/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/jumpcloud/data_stream/events/elasticsearch/ingest_pipeline/default.yml index bc16480be8c..eaa50122274 100644 --- a/packages/jumpcloud/data_stream/events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/jumpcloud/data_stream/events/elasticsearch/ingest_pipeline/default.yml @@ -2,9 +2,11 @@ description: Pipeline for JumpCloud Events processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -16,9 +18,11 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - script: + tag: script_e2ccbdd7 description: Drops null/empty values recursively. lang: painless source: | @@ -36,12 +40,15 @@ processors: } dropEmptyFields(ctx); - fail: + tag: fail_d1247e5f if: ctx.json == null || !(ctx.json instanceof Map) message: missing json object in input document - rename: + tag: rename_json_to_jumpcloud_event_d676b966 field: json target_field: jumpcloud.event - fingerprint: + tag: fingerprint_3fc41178 description: "Generate a unique document ID using the unique JumpCloud event ID and timestamp" fields: - jumpcloud.event.id @@ -50,46 +57,55 @@ processors: method: MurmurHash3 ignore_missing: true - date: + tag: date_jumpcloud_event_timestamp_c832a28c field: jumpcloud.event.timestamp formats: ["ISO8601"] if: ctx.jumpcloud?.event?.timestamp != null && ctx.jumpcloud?.event?.timestamp != "" - convert: + tag: convert_jumpcloud_event_client_ip_to_client_ip_674a2272 field: jumpcloud.event.client_ip type: ip target_field: client.ip ignore_missing: true - convert: + tag: convert_jumpcloud_event_src_ip_to_source_ip_0b3d327d field: jumpcloud.event.src_ip type: ip target_field: source.ip ignore_missing: true - append: + tag: append_event_category_1a04a01a field: event.category value: authentication if: ctx.jumpcloud?.event?.event_type != null && ['ldap_bind', 'user_login_attempt', 'sso_auth', 'admin_login_attempt'].contains(ctx.jumpcloud.event.event_type) allow_duplicates: false - append: + tag: append_event_type_e9398a56 field: event.type value: - info if: ctx.jumpcloud?.event?.event_type != null && ['ldap_bind', 'user_login_attempt', 'sso_auth', 'admin_login_attempt'].contains(ctx.jumpcloud.event.event_type) allow_duplicates: false - append: + tag: append_event_category_1bc99d0a field: event.category value: iam if: ctx.jumpcloud?.event?.event_type != null && ['user_password_warning_email', 'user_password_reset_request', 'user_activation_email', 'user_unlocked'].contains(ctx.jumpcloud.event.event_type) allow_duplicates: false - append: + tag: append_event_type_f53c4aff field: event.type value: - info if: ctx.jumpcloud?.event?.event_type != null && ['user_password_warning_email', 'user_password_reset_request', 'user_activation_email', 'user_unlocked'].contains(ctx.jumpcloud.event.event_type) allow_duplicates: false - set: + tag: set_event_id_c308d5dc copy_from: jumpcloud.event.id field: event.id ignore_empty_value: true - set: + tag: set_message_1d0a92e4 copy_from: jumpcloud.event.message field: message ignore_empty_value: true @@ -115,97 +131,119 @@ processors: ignore_missing: true ### utilise host fields if they exist - set: + tag: set_host_id_d195da7b copy_from: jumpcloud.event.system.id field: host.id ignore_empty_value: true - set: + tag: set_host_hostname_ae3a3b27 copy_from: jumpcloud.event.system.hostname field: host.hostname ignore_empty_value: true ### utilise user agent fields if they exist - set: + tag: set_source_user_id_8cf8bbb1 copy_from: jumpcloud.event.initiated_by.id field: source.user.id ignore_empty_value: true - set: + tag: set_source_user_email_64781173 copy_from: jumpcloud.event.initiated_by.email field: source.user.email ignore_empty_value: true - set: + tag: set_source_user_name_e52ffab2 copy_from: jumpcloud.event.initiated_by.username field: source.user.name ignore_empty_value: true - set: + tag: set_user_name_934915fc copy_from: jumpcloud.event.username field: user.name ignore_empty_value: true ### utilise user agent fields if they exist - set: + tag: set_user_agent_device_name_9c984054 copy_from: jumpcloud.event.useragent.device field: user_agent.device.name ignore_empty_value: true - set: + tag: set_user_agent_name_e0721eed copy_from: jumpcloud.event.useragent.name field: user_agent.name ignore_empty_value: true - set: + tag: set_user_agent_version_369b0727 copy_from: jumpcloud.event.useragent.version field: user_agent.version ignore_empty_value: true - set: + tag: set_user_agent_os_name_96303e4a copy_from: jumpcloud.event.useragent.os_name field: user_agent.os.name ignore_empty_value: true - set: + tag: set_user_agent_os_full_02866b62 copy_from: jumpcloud.event.useragent.os_full field: user_agent.os.full ignore_empty_value: true - set: + tag: set_user_agent_os_version_e1252192 copy_from: jumpcloud.event.useragent.os_version field: user_agent.os.version ignore_empty_value: true ### event action - set: + tag: set_event_action_7eccfd51 copy_from: jumpcloud.event.event_type field: event.action ignore_empty_value: true ### event module - set: + tag: set_event_module_e99896d1 copy_from: jumpcloud.event.service field: event.module ignore_empty_value: true ### process name - set: + tag: set_process_name_d3b9e9ef copy_from: jumpcloud.event.process_name field: process.name ignore_empty_value: true ### event outcome - set: + tag: set_event_outcome_82c7155e field: event.outcome value: unknown - set: + tag: set_event_outcome_d3fb9fd5 field: event.outcome value: success if: ctx.jumpcloud?.event?.sso_token_success == true - set: + tag: set_event_outcome_d72217f5 field: event.outcome value: failure if: ctx.jumpcloud?.event?.sso_token_success == false - set: + tag: set_event_outcome_adf50dab field: event.outcome value: success if: ctx.jumpcloud?.event?.success == true - set: + tag: set_event_outcome_4ea51fb7 field: event.outcome value: failure if: ctx.jumpcloud?.event?.success == false ### geoip enrichment - geoip: + tag: geoip_source_ip_to_source_geo_859b1031 field: source.ip target_field: source.geo ignore_missing: true if: ctx.source?.geo == null && ctx.source?.ip != null - geoip: + tag: geoip_source_ip_to_source_as_5a30a433 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -215,19 +253,23 @@ processors: ignore_missing: true if: ctx.source?.ip != null - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - geoip: + tag: geoip_client_ip_to_client_geo_c8162a11 field: client.ip target_field: client.geo ignore_missing: true if: ctx.client?.geo == null && ctx.client?.ip != null - geoip: + tag: geoip_client_ip_to_client_as_0a1b2b55 database_file: GeoLite2-ASN.mmdb field: client.ip target_field: client.as @@ -237,65 +279,78 @@ processors: ignore_missing: true if: ctx.client?.ip != null - rename: + tag: rename_client_as_asn_to_client_as_number_a6e30d01 field: client.as.asn target_field: client.as.number ignore_missing: true - rename: + tag: rename_client_as_organization_name_to_client_as_organization_name_817a526f field: client.as.organization_name target_field: client.as.organization.name ignore_missing: true ### relateds - append: + tag: append_related_ip_121e62cf field: related.ip value: "{{{jumpcloud.event.jumpcloud_protect_device.ip}}}" if: "ctx.jumpcloud?.event?.jumpcloud_protect_device?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: "ctx.source?.ip != null" allow_duplicates: false - append: + tag: append_client_ip_a848f9e9 field: client.ip value: "{{{client.ip}}}" if: "ctx.client?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: "ctx.destination?.ip != null" allow_duplicates: false - append: + tag: append_related_ip_c3acf835 field: related.ip value: "{{{server.ip}}}" if: "ctx.server?.ip != null" allow_duplicates: false - append: + tag: append_related_user_f745a124 field: related.user value: "{{{source.user.name}}}" if: "ctx.source?.user?.name != null" allow_duplicates: false - append: + tag: append_related_user_8f37bc82 field: related.user value: "{{{destination.user.name}}}" if: "ctx.destination?.user?.name != null" allow_duplicates: false - append: + tag: append_related_hosts_aa05bed9 field: related.hosts value: "{{{destination.address}}}" if: "ctx.destination?.address != null" allow_duplicates: false - append: + tag: append_related_hosts_439b2767 field: related.hosts value: "{{{source.address}}}" if: "ctx.source?.address != null" allow_duplicates: false - remove: + tag: remove_615a3966 description: Cleanup temporary and other garbage fields field: - _tmp ignore_missing: true - remove: + tag: remove_4d2032e2 description: Remove jumpcloud fields that are no longer needed as they were mapped to ECS, unless instructed not to field: - jumpcloud.event.timestamp diff --git a/packages/jumpcloud/manifest.yml b/packages/jumpcloud/manifest.yml index bd38580fb05..ca02e2079c5 100644 --- a/packages/jumpcloud/manifest.yml +++ b/packages/jumpcloud/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.3.2" name: jumpcloud title: "JumpCloud" -version: "1.22.2" +version: "1.22.3" description: "Collect logs from JumpCloud Directory as a Service" type: integration categories: diff --git a/packages/keeper_security_siem_integration/changelog.yml b/packages/keeper_security_siem_integration/changelog.yml index 11e39d8f4a1..7a699c2f741 100644 --- a/packages/keeper_security_siem_integration/changelog.yml +++ b/packages/keeper_security_siem_integration/changelog.yml @@ -1,3 +1,8 @@ +- version: "0.1.1" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "0.1.0" changes: - description: Initial release of Keeper Security agentless integration diff --git a/packages/keeper_security_siem_integration/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/keeper_security_siem_integration/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 429e9d0f368..b899a55c2a6 100644 --- a/packages/keeper_security_siem_integration/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/keeper_security_siem_integration/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -4,23 +4,28 @@ description: "Agentless pipeline for processing Keeper Security audit events" processors: # Set ECS version - set: + tag: set_ecs_version_f5923549 field: ecs.version value: "8.17.0" # Set basic event fields - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_category_78b61ddc field: event.category value: ["authentication", "web"] - set: + tag: set_event_type_60726f2a field: event.type value: ["access", "info"] # Handle timestamp conversion - support multiple formats - date: + tag: date_timestamp_to_timestamp_b79e2b56 field: timestamp target_field: '@timestamp' formats: @@ -34,65 +39,75 @@ processors: if: ctx.timestamp != null on_failure: - append: + tag: append_error_message_0c68e68d field: error.message value: "Failed to parse timestamp: {{{timestamp}}}" - + - set: + tag: set_user_name_dd8f018f field: user.name copy_from: username ignore_empty_value: true description: "Set user name from username" - set: - field: user.email + tag: set_user_email_440d48a9 + field: user.email copy_from: username if: ctx.username != null && ctx.username.contains("@") description: "If username looks like email, set user.email" - set: + tag: set_source_ip_0ebd235b field: source.ip copy_from: remote_address ignore_empty_value: true description: "Set source IP from remote_address" - geoip: + tag: geoip_source_ip_to_source_geo_d3aff80c field: source.ip target_field: source.geo on_failure: - append: + tag: append_error_message_8118ef25 field: error.message value: "GeoIP lookup failed for source.ip={{{source.ip}}}" description: "GeoIP enrichment -> ECS source.geo.*" - set: + tag: set_event_action_7e62b17b field: event.action copy_from: audit_event ignore_empty_value: true description: "Set event action from audit_event" - convert: + tag: convert_enterprise_id_to_organization_id_4d2a7a07 field: enterprise_id type: string target_field: organization.id ignore_missing: true ignore_failure: true description: "Convert enterprise_id to organization fields" - + - set: + tag: set_user_agent_original_0ba25c09 field: user_agent.original value: "Keeper/{{{client_version}}}" ignore_empty_value: true description: "Set user agent from client_version" - # Build related fields for correlation - append: + tag: append_related_user_837e080f field: related.user value: "{{{user.name}}}" if: ctx.user?.name != null allow_duplicates: false - + - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: ctx.source?.ip != null @@ -100,16 +115,19 @@ processors: # Set event outcome based on category or audit_event - set: + tag: set_event_outcome_bf4fe1b8 field: event.outcome value: "success" if: ctx.category != null && !ctx.category.contains("fail") - + - set: + tag: set_event_outcome_5831a510 field: event.outcome value: "failure" if: ctx.category != null && ctx.category.contains("fail") - remove: + tag: remove_timestamp_29d0d912 field: timestamp ignore_missing: true description: "Clean up - remove original timestamp to avoid confusion" @@ -120,6 +138,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/keeper_security_siem_integration/manifest.yml b/packages/keeper_security_siem_integration/manifest.yml index c380f8ade03..fc4216a1ac8 100644 --- a/packages/keeper_security_siem_integration/manifest.yml +++ b/packages/keeper_security_siem_integration/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.0 name: keeper title: "Keeper Security" -version: 0.1.0 +version: 0.1.1 description: > Keeper Security agentless integration for collecting audit events directly via Elasticsearch Bulk API. No agents required - Keeper pushes data directly to Elasticsearch. diff --git a/packages/keycloak/changelog.yml b/packages/keycloak/changelog.yml index ec74eb1338f..78c710510a8 100644 --- a/packages/keycloak/changelog.yml +++ b/packages/keycloak/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.32.1" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.32.0" changes: - description: Set 'elastic' owner type. diff --git a/packages/keycloak/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/keycloak/data_stream/log/elasticsearch/ingest_pipeline/default.yml index cce56d6b9e4..f3ae86ca16a 100644 --- a/packages/keycloak/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/keycloak/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing keycloak logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - rename: @@ -11,19 +12,23 @@ processors: if: ctx.event?.original == null tag: rename_event_original - json: + tag: json_event_original_to_json_cac66847 field: event.original target_field: json ignore_failure: true - dot_expander: + tag: dot_expander_*_add55d98 field: "*" path: json if: ctx.json != null - rename: + tag: rename_event_original_to_json_event_original_a7fe9915 field: event.original target_field: json.event.original ignore_missing: true if: ctx.json != null - remove: + tag: remove__tmp_tz_offset_705bc198 field: _tmp.tz_offset if: ctx.json != null - grok: diff --git a/packages/keycloak/data_stream/log/elasticsearch/ingest_pipeline/events.yml b/packages/keycloak/data_stream/log/elasticsearch/ingest_pipeline/events.yml index b20aa5407b0..5dbd049d67a 100644 --- a/packages/keycloak/data_stream/log/elasticsearch/ingest_pipeline/events.yml +++ b/packages/keycloak/data_stream/log/elasticsearch/ingest_pipeline/events.yml @@ -10,88 +10,108 @@ processors: ignore_missing: true tag: kv_message - rename: + tag: rename_json_type_to_keycloak_login_type_57d819de field: json.type target_field: keycloak.login.type ignore_missing: true - set: + tag: set_event_outcome_49ff15b2 field: event.outcome value: success if: ctx.keycloak?.login?.type != null && ctx.keycloak.login.type.equalsIgnoreCase('login') - set: + tag: set_event_outcome_2daf08fa field: event.outcome value: failure if: ctx.keycloak?.login?.type != null && ctx.keycloak.login.type.equalsIgnoreCase('login_error') - set: + tag: set_event_outcome_6f947b7c field: event.outcome value: unknown if: ctx.event?.outcome == null - rename: + tag: rename_json_operationType_to_keycloak_admin_operation_2841a942 field: json.operationType target_field: keycloak.admin.operation ignore_missing: true - rename: + tag: rename_json_resourceType_to_keycloak_admin_resource_type_e5182722 field: json.resourceType target_field: keycloak.admin.resource.type ignore_missing: true - rename: + tag: rename_json_resourcePath_to_keycloak_admin_resource_path_e55d3ece field: json.resourcePath target_field: keycloak.admin.resource.path ignore_missing: true - set: + tag: set_keycloak_event_type_a3def70f field: keycloak.event_type value: login if: ctx.keycloak?.login != null - set: + tag: set_keycloak_event_type_0b9ffb43 field: keycloak.event_type value: admin if: ctx.keycloak?.admin != null - set: + tag: set_event_code_44eb2ca7 field: event.code value: "{{{keycloak.admin.operation}}}-{{{keycloak.admin.resource.type}}}" if: ctx.keycloak?.admin != null - set: + tag: set_event_action_211ad612 field: event.action copy_from: event.code ignore_empty_value: true if: ctx.keycloak?.admin != null - rename: + tag: rename_json_error_to_event_code_84b48bf7 field: json.error target_field: event.code ignore_missing: true if: ctx.keycloak?.login != null && ctx.event?.code == null - set: + tag: set_event_action_e881cd2d field: event.action copy_from: keycloak.login.type ignore_empty_value: true - rename: + tag: rename_json_realmId_to_keycloak_realm_id_8b4ca0b7 field: json.realmId target_field: keycloak.realm.id ignore_missing: true - rename: + tag: rename_json_clientId_to_keycloak_client_id_5ee27ce7 field: json.clientId target_field: keycloak.client.id ignore_missing: true if: ctx.json?.clientId != "null" - rename: + tag: rename_json_userId_to_user_id_e1c26f9a field: json.userId target_field: user.id ignore_missing: true if: ctx.json?.userId != "null" - rename: + tag: rename_json_ipAddress_to_source_address_013aabe4 field: json.ipAddress target_field: source.address ignore_missing: true - convert: + tag: convert_source_address_to_source_ip_f5632a20 field: source.address target_field: source.ip type: ip ignore_failure: true ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -100,137 +120,166 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_json_redirect_uri_to_keycloak_login_redirect_uri_044bc454 field: json.redirect_uri target_field: keycloak.login.redirect_uri ignore_missing: true - uri_parts: + tag: uri_parts_keycloak_login_redirect_uri_b7f273c6 field: keycloak.login.redirect_uri ignore_failure: true - rename: + tag: rename_json_auth_method_to_keycloak_login_auth_method_f81f9bc4 field: json.auth_method target_field: keycloak.login.auth_method ignore_missing: true - rename: + tag: rename_json_auth_type_to_keycloak_login_auth_type_83580590 field: json.auth_type target_field: keycloak.login.auth_type ignore_missing: true - rename: + tag: rename_json_code_id_to_keycloak_login_code_id_b9c73768 field: json.code_id target_field: keycloak.login.code_id ignore_missing: true - rename: + tag: rename_json_username_to_user_name_c9e5d2a6 field: json.username target_field: user.name ignore_missing: true - rename: + tag: rename_json_authSessionParentId_to_keycloak_login_auth_session_parent_id_64031e91 field: json.authSessionParentId target_field: keycloak.login.auth_session_parent_id ignore_missing: true - rename: + tag: rename_json_authSessionTabId_to_keycloak_login_auth_session_tab_id_7497766f field: json.authSessionTabId target_field: keycloak.login.auth_session_tab_id ignore_missing: true - rename: + tag: rename_json_impersonator_realm_to_keycloak_impersonator_realm_7eef9d92 field: json.impersonator_realm target_field: keycloak.impersonator_realm ignore_missing: true if: ctx.json?.impersonator_realm != "null" - rename: + tag: rename_json_impersonator_to_keycloak_impersonator_936d013a field: json.impersonator target_field: keycloak.impersonator ignore_missing: true if: ctx.json?.impersonator != "null" - rename: + tag: rename_json_sessionId_to_keycloak_session_id_2f9f5e46 field: json.sessionId target_field: keycloak.session.id ignore_missing: true if: ctx.json?.sessionId != "null" - grok: field: keycloak.admin.resource.path - patterns: + patterns: - 'users/%{UUID:user.target.id}' - 'groups/%{UUID:group.id}' ignore_failure: true ignore_missing: true tag: grok_admin_resource_path - set: + tag: set_event_kind_de80643c field: event.kind value: event - append: + tag: append_event_category_c8bca37c field: event.category value: - authentication - if: ctx.keycloak?.login != null + if: ctx.keycloak?.login != null - append: + tag: append_event_type_8b66b73a field: event.type value: - info - append: + tag: append_event_type_13c23515 field: event.type value: - start - if: ctx.keycloak?.login?.type == 'LOGIN' + if: ctx.keycloak?.login?.type == 'LOGIN' - append: + tag: append_event_type_f79c585b field: event.type value: - end - if: ctx.keycloak?.login?.type == 'LOGOUT' + if: ctx.keycloak?.login?.type == 'LOGOUT' - append: + tag: append_event_category_a5b13e35 field: event.category value: - iam - if: ctx.keycloak?.admin != null + if: ctx.keycloak?.admin != null - append: + tag: append_event_type_d66c1ef3 field: event.type value: - admin - if: ctx.keycloak?.admin != null + if: ctx.keycloak?.admin != null - append: + tag: append_event_type_a9ec476c field: event.type value: - creation if: ctx.keycloak?.admin?.operation == "CREATE" - append: + tag: append_event_type_71f8f290 field: event.type value: - change if: ctx.keycloak?.admin?.operation == "UPDATE" - append: + tag: append_event_type_81dc02f0 field: event.type value: - deletion if: ctx.keycloak?.admin?.operation == "DELETE" - append: + tag: append_event_type_c1a6ca94 field: event.type value: - group if: ctx.keycloak?.admin?.resource == "GROUP" - append: + tag: append_event_type_e62fcfce field: event.type value: - user if: ctx.keycloak?.admin?.resource == "USER" - append: + tag: append_related_ip_86933b04 field: related.ip value: "{{{source.ip}}}" if: ctx.source?.ip != null - append: + tag: append_related_user_505d83e6 field: related.user value: "{{{user.id}}}" if: ctx.user?.id != null - append: + tag: append_related_user_0bb97893 field: related.user value: "{{{user.target.id}}}" if: ctx.user?.target?.id != null - append: + tag: append_related_hosts_12369daa field: related.hosts value: "{{{url.domain}}}" if: ctx.url?.domain != null @@ -248,3 +297,7 @@ on_failure: - set: field: event.kind value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/keycloak/manifest.yml b/packages/keycloak/manifest.yml index 4d3eba531b1..5faa63db475 100644 --- a/packages/keycloak/manifest.yml +++ b/packages/keycloak/manifest.yml @@ -1,6 +1,6 @@ name: keycloak title: Keycloak -version: "1.32.0" +version: "1.32.1" description: Collect logs from Keycloak with Elastic Agent. type: integration format_version: "3.0.3" diff --git a/packages/lastpass/changelog.yml b/packages/lastpass/changelog.yml index 4de8284adcc..daafb62902c 100644 --- a/packages/lastpass/changelog.yml +++ b/packages/lastpass/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.23.3" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.23.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/lastpass/data_stream/detailed_shared_folder/elasticsearch/ingest_pipeline/default.yml b/packages/lastpass/data_stream/detailed_shared_folder/elasticsearch/ingest_pipeline/default.yml index 113e5b58b31..26b8d49af3c 100644 --- a/packages/lastpass/data_stream/detailed_shared_folder/elasticsearch/ingest_pipeline/default.yml +++ b/packages/lastpass/data_stream/detailed_shared_folder/elasticsearch/ingest_pipeline/default.yml @@ -2,9 +2,11 @@ description: Pipeline for processing Detailed Shared Folder logs. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -16,19 +18,24 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] - set: + tag: set_event_kind_871e09cf field: event.kind value: state - rename: + tag: rename_json_sharedfoldername_to_lastpass_detailed_shared_folder_name_e272d13b field: json.sharedfoldername target_field: lastpass.detailed_shared_folder.name ignore_missing: true - convert: + tag: convert_json_deleted_to_lastpass_detailed_shared_folder_deleted_a8b5f3fe field: json.deleted target_field: lastpass.detailed_shared_folder.deleted if: ctx.json?.deleted != '' @@ -36,9 +43,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_7a3a7145 field: error.message value: '{{{ _ingest.on_failure_message }}}' - convert: + tag: convert_json_score_to_lastpass_detailed_shared_folder_score_1c783122 field: json.score target_field: lastpass.detailed_shared_folder.score if: ctx.json?.score != '' @@ -46,21 +55,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_0bc2580f field: error.message value: '{{{ _ingest.on_failure_message }}}' - rename: + tag: rename_json_id_to_lastpass_detailed_shared_folder_shared_folder_id_679caa80 field: json.id target_field: lastpass.detailed_shared_folder.shared_folder.id ignore_missing: true - rename: + tag: rename_json_users_username_to_lastpass_detailed_shared_folder_user_name_d11bae96 field: json.users.username target_field: lastpass.detailed_shared_folder.user.name ignore_missing: true - set: + tag: set_user_email_affa1168 field: user.email copy_from: lastpass.detailed_shared_folder.user.name ignore_empty_value: true - convert: + tag: convert_json_users_superadmin_to_lastpass_detailed_shared_folder_user_super_admin_121bfbf0 field: json.users.superadmin target_field: lastpass.detailed_shared_folder.user.super_admin if: ctx.json?.users?.superadmin != '' @@ -68,9 +82,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_54e13303 field: error.message value: '{{{ _ingest.on_failure_message }}}' - convert: + tag: convert_json_users_readonly_to_lastpass_detailed_shared_folder_user_read_only_3779f3a0 field: json.users.readonly target_field: lastpass.detailed_shared_folder.user.read_only if: ctx.json?.users?.readonly != '' @@ -78,9 +94,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_1edb1f73 field: error.message value: '{{{ _ingest.on_failure_message }}}' - convert: + tag: convert_json_users_give_to_lastpass_detailed_shared_folder_user_give_f0ad47e6 field: json.users.give target_field: lastpass.detailed_shared_folder.user.give if: ctx.json?.users?.give != '' @@ -88,9 +106,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a6665d93 field: error.message value: '{{{ _ingest.on_failure_message }}}' - convert: + tag: convert_json_users_can_administer_to_lastpass_detailed_shared_folder_user_can_administer_f6a5a702 field: json.users.can_administer target_field: lastpass.detailed_shared_folder.user.can_administer if: ctx.json?.users?.can_administer != '' @@ -98,29 +118,35 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2407f1c7 field: error.message value: '{{{ _ingest.on_failure_message }}}' - rename: + tag: rename_json_users_sites_to_lastpass_detailed_shared_folder_user_site_ee573ae0 field: json.users.sites target_field: lastpass.detailed_shared_folder.user.site ignore_missing: true - append: + tag: append_related_user_36d4b55a field: related.user value: '{{{user.email}}}' if: ctx.user?.email != null allow_duplicates: false ignore_failure: true - remove: + tag: remove_e5a80934 field: - json ignore_missing: true - remove: + tag: remove_b906fe69 if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - lastpass.detailed_shared_folder.user.name ignore_failure: true ignore_missing: true - script: + tag: script_e2ccbdd7 description: Drops null/empty values recursively. lang: painless source: | diff --git a/packages/lastpass/data_stream/event_report/elasticsearch/ingest_pipeline/default.yml b/packages/lastpass/data_stream/event_report/elasticsearch/ingest_pipeline/default.yml index 17cbde158ee..f5d706634dd 100644 --- a/packages/lastpass/data_stream/event_report/elasticsearch/ingest_pipeline/default.yml +++ b/packages/lastpass/data_stream/event_report/elasticsearch/ingest_pipeline/default.yml @@ -2,9 +2,11 @@ description: Pipeline for processing Event Report logs. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -16,9 +18,11 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - fingerprint: + tag: fingerprint_b44d2872 fields: - json.IP_Address - json.Username @@ -28,20 +32,25 @@ processors: target_field: '_id' ignore_missing: true - rename: + tag: rename_json_Action_to_lastpass_event_report_action_69f911e2 field: json.Action target_field: lastpass.event_report.action ignore_missing: true - set: + tag: set_event_action_8bf755cd field: event.action copy_from: lastpass.event_report.action ignore_empty_value: true - lowercase: + tag: lowercase_event_action_9334b869 field: event.action ignore_missing: true - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_category_d2f3d550 field: event.category if: >- ctx.event?.action != null && ( @@ -54,6 +63,7 @@ processors: ) value: [authentication] - set: + tag: set_event_category_a4de1d17 field: event.category if: >- ctx.event?.action != null && ( @@ -63,6 +73,7 @@ processors: ) value: [configuration] - set: + tag: set_event_category_7e8cfc56 field: event.category if: >- ctx.event?.action != null && ( @@ -79,6 +90,7 @@ processors: ) value: [iam] - set: + tag: set_event_type_8dad002d field: event.type if: >- ctx.event?.action != null && ( @@ -90,10 +102,12 @@ processors: ) value: [start] - set: + tag: set_event_type_ef084da0 field: event.type if: ctx.event?.action?.contains('make admin') == true value: [admin] - set: + tag: set_event_type_bee16e41 field: event.type if: >- ctx.event?.action != null && ( @@ -110,6 +124,7 @@ processors: ) value: [change] - set: + tag: set_event_type_5418f29c field: event.type if: >- ctx.event?.action != null && ( @@ -123,6 +138,7 @@ processors: ) value: [creation] - set: + tag: set_event_type_c14d46f8 field: event.type if: >- ctx.event?.action != null && ( @@ -137,6 +153,7 @@ processors: ) value: [deletion] - set: + tag: set_event_type_3b9297fa field: event.type if: >- ctx.event?.action != null && ( @@ -145,6 +162,7 @@ processors: ) value: [group, creation] - set: + tag: set_event_type_b08821ea field: event.type if: >- ctx.event?.action != null && ( @@ -158,6 +176,7 @@ processors: ) value: [info] - set: + tag: set_event_outcome_1f489e0f field: event.outcome if: >- ctx.event?.action != null && ( @@ -201,10 +220,12 @@ processors: ) value: success - set: + tag: set_event_outcome_c20d4e51 field: event.outcome if: ctx.event?.action?.contains('failed login attempt') == true value: failure - set: + tag: set_event_outcome_42e772e3 field: event.outcome if: >- ctx.event?.action != null && ( @@ -215,15 +236,18 @@ processors: # Set timezone based on config or locale. Defaults to US/Eastern due to # https://support.lastpass.com/s/document-item?language=en_US&bundleId=lastpass&topicId=LastPass/api_event_reporting.html&_LANG=enus. - rename: + tag: rename__conf_tz_offset_to_event_timezone_1561f981 field: _conf.tz_offset target_field: event.timezone if: ctx._conf?.tz_offset != null ignore_failure: true - set: + tag: set_event_timezone_5dd4d807 field: event.timezone value: US/Eastern override: false - date: + tag: date_json_Time_to_lastpass_event_report_time_33f87d37 field: json.Time if: ctx.json?.Time != null && ctx.json.Time != '' && ctx.event?.timezone != null target_field: lastpass.event_report.time @@ -235,13 +259,16 @@ processors: - yyyy-MM-dd HH:mm:ssZZZZ on_failure: - append: + tag: append_error_message_e0251492 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_timestamp_c9be534a field: '@timestamp' copy_from: lastpass.event_report.time ignore_empty_value: true - convert: + tag: convert_json_IP_Address_to_lastpass_event_report_ip_7f8f2c37 field: json.IP_Address target_field: lastpass.event_report.ip if: ctx.json?.IP_Address != '' @@ -249,31 +276,38 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_46f3daaa field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_source_ip_48d07e36 field: source.ip copy_from: lastpass.event_report.ip ignore_empty_value: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - rename: + tag: rename_json_Username_to_lastpass_event_report_user_name_3967cfd7 field: json.Username target_field: lastpass.event_report.user_name ignore_missing: true - rename: + tag: rename_json_Data_to_lastpass_event_report_data_original_aad0292d field: json.Data target_field: lastpass.event_report.data.original ignore_missing: true - split: + tag: split_lastpass_event_report_data_original_to_lastpass_event_report_data_deleted_site_344765c4 if: ctx.event?.action?.contains('deleted sites') == true field: lastpass.event_report.data.original separator: ',' target_field: lastpass.event_report.data.deleted_site ignore_failure: true - split: + tag: split_lastpass_event_report_data_original_to_lastpass_event_report_data_user_email_f31adb40 if: >- ctx.event?.action != null && ( ctx.event.action.contains('employee invited') || @@ -285,6 +319,7 @@ processors: target_field: lastpass.event_report.data.user_email ignore_failure: true - script: + tag: script_c42b5c04 description: Separate Shared Folder Name and User Email with comma(',') in Limit Shared Folder Event Type. lang: painless source: | @@ -294,24 +329,28 @@ processors: ctx._temp = str; } - grok: + tag: grok_lastpass_event_report_data_original_4f05c0d8 field: lastpass.event_report.data.original if: ctx.event?.action?.contains('log in') == true patterns: - "^%{GREEDYDATA:lastpass.event_report.data.login_site}$" ignore_failure: true - grok: + tag: grok_lastpass_event_report_data_original_7fdcb29f field: lastpass.event_report.data.original if: ctx.event?.action?.contains('saml login') == true patterns: - "^%{GREEDYDATA:lastpass.event_report.data.saml_login}$" ignore_failure: true - grok: + tag: grok_lastpass_event_report_data_original_904310e2 field: lastpass.event_report.data.original if: ctx.event?.action?.contains('failed login attempt') == true patterns: - "^%{GREEDYDATA:lastpass.event_report.data.failed_login}$" ignore_failure: true - grok: + tag: grok_lastpass_event_report_data_original_0f206e9d field: lastpass.event_report.data.original if: >- ctx.event?.action != null && ( @@ -325,12 +364,14 @@ processors: - "^%{EMAILADDRESS:lastpass.event_report.data.user_email}$" ignore_failure: true - grok: + tag: grok_lastpass_event_report_data_original_42574f3d field: lastpass.event_report.data.original if: ctx.event?.action?.contains('site added') == true patterns: - "^%{GREEDYDATA:lastpass.event_report.data.added_site}$" ignore_failure: true - grok: + tag: grok_lastpass_event_report_data_original_399541cc field: lastpass.event_report.data.original if: >- ctx.event?.action != null && ( @@ -341,6 +382,7 @@ processors: - "^%{DATA:lastpass.event_report.data.shared_folder_name}$" ignore_failure: true - grok: + tag: grok_lastpass_event_report_data_original_e289fb20 field: lastpass.event_report.data.original if: >- ctx.event?.action != null && ( @@ -352,48 +394,56 @@ processors: - '^Secure Note\s+\(%{DATA:lastpass.event_report.data.secure_note}\)\s+from\s+%{DATA:lastpass.event_report.data.shared_folder_name}$' ignore_failure: true - grok: + tag: grok_lastpass_event_report_data_original_3187d099 field: lastpass.event_report.data.original if: ctx.event?.action?.contains('edit secure note') == true patterns: - '^Secure Note\s+\(%{DATA:lastpass.event_report.data.secure_note}\)$' ignore_failure: true - grok: + tag: grok_lastpass_event_report_data_original_d6a637ec field: lastpass.event_report.data.original if: ctx.event?.action?.contains('add to shared folder') == true patterns: - "^'%{DATA:lastpass.event_report.data.shared_folder_name}'\\s+'%{EMAILADDRESS:lastpass.event_report.data.user_email}'$" ignore_failure: true - grok: + tag: grok_lastpass_event_report_data_original_d8dd77e0 field: lastpass.event_report.data.original if: ctx.event?.action?.contains('create group') == true patterns: - "^'%{DATA:lastpass.event_report.data.group_name}'$" ignore_failure: true - grok: + tag: grok_lastpass_event_report_data_original_dbd8bfcb field: lastpass.event_report.data.original if: ctx.event?.action?.contains('adding user to group') == true patterns: - '^%{EMAILADDRESS:lastpass.event_report.data.user_email}\s+\-\s+%{DATA:lastpass.event_report.data.group_name}$' ignore_failure: true - grok: + tag: grok_lastpass_event_report_data_original_acdda5c0 field: lastpass.event_report.data.original if: ctx.event?.action?.contains('created lastpass account') == true patterns: - '^%{EMAILADDRESS:lastpass.event_report.data.user_email}\s*-Shared-\s*%{DATA:lastpass.event_report.data.shared_folder_name}$' ignore_failure: true - grok: + tag: grok_lastpass_event_report_data_original_100a7758 field: lastpass.event_report.data.original if: ctx.event?.action?.contains('update folder permissions') == true patterns: - "^'%{DATA:lastpass.event_report.data.shared_folder_name}'\\s+'%{EMAILADDRESS:lastpass.event_report.data.user_email}'\\s+'Read only:%{DATA:lastpass.event_report.data.shared_folder_user_permissions.read_only}\\s+Admin:%{DATA:lastpass.event_report.data.shared_folder_user_permissions.admin}\\s+Hide PW:%{DATA:lastpass.event_report.data.shared_folder_user_permissions.hide_password}'$" ignore_failure: true - grok: + tag: grok_lastpass_event_report_data_original_9100b99d field: lastpass.event_report.data.original if: ctx.event?.action?.contains('renamed shared folder') == true patterns: - "^'%{DATA:lastpass.event_report.data.shared_folder_name}'\\s+'%{DATA:lastpass.event_report.data.renamed_shared_folder_name}'$" ignore_failure: true - grok: + tag: grok_lastpass_event_report_data_original_fc136b7c field: lastpass.event_report.data.original if: ctx.event?.action?.contains('move to shared folder') == true patterns: @@ -401,6 +451,7 @@ processors: - '^%{GREEDYDATA:lastpass.event_report.data.site}\s+to\s+%{DATA:lastpass.event_report.data.shared_folder_name}$' ignore_failure: true - grok: + tag: grok_lastpass_event_report_data_original_a685cb63 field: lastpass.event_report.data.original if: ctx.event?.action?.contains('move from shared folder') == true patterns: @@ -409,6 +460,7 @@ processors: - '^%{GREEDYDATA:lastpass.event_report.data.site}\s+from\s+%{DATA:lastpass.event_report.data.shared_folder_name}$' ignore_failure: true - grok: + tag: grok_lastpass_event_report_data_original_c2e95fcf field: lastpass.event_report.data.original if: ctx.event?.action?.contains('delete shared sites') == true patterns: @@ -416,22 +468,26 @@ processors: - '^%{GREEDYDATA:lastpass.event_report.data.deleted_site}\s+from\s+%{DATA:lastpass.event_report.data.shared_folder_name}$' ignore_failure: true - grok: + tag: grok__temp_0084a285 field: _temp if: ctx.event?.action?.contains('limit shared folder') == true patterns: - '^%{DATA:lastpass.event_report.data.shared_folder_name},%{EMAILADDRESS:lastpass.event_report.data.user_email}$' ignore_failure: true - grok: + tag: grok_lastpass_event_report_data_original_81ce440c field: lastpass.event_report.data.original if: ctx.event?.action?.contains('removed from shared folder') == true patterns: - "^'%{DATA:lastpass.event_report.data.shared_folder_name}'\\s+'%{EMAILADDRESS:lastpass.event_report.data.user_email}'$" ignore_failure: true - set: + tag: set_user_group_name_8e2b6cfd field: user.group.name copy_from: lastpass.event_report.data.group_name ignore_empty_value: true - foreach: + tag: foreach_lastpass_event_report_data_user_email_8f5196ee if: >- ctx.event?.action != null && ( ctx.event.action.contains('employee invited') || @@ -448,6 +504,7 @@ processors: ignore_missing: true ignore_failure: true - append: + tag: append_user_email_a0e20214 if: >- ctx.event?.action == null || !( ctx.event.action.contains('employee invited') || @@ -460,12 +517,14 @@ processors: allow_duplicates: false ignore_failure: true - append: + tag: append_user_email_b9a7f4e6 field: user.email value: - '{{{lastpass.event_report.user_name}}}' allow_duplicates: false ignore_failure: true - foreach: + tag: foreach_user_email_6d2203b4 field: user.email processor: append: @@ -476,17 +535,20 @@ processors: ignore_missing: true ignore_failure: true - append: + tag: append_related_ip_30d15214 field: related.ip value: '{{{source.ip}}}' if: ctx.source?.ip != null allow_duplicates: false ignore_failure: true - remove: + tag: remove_ef4e960b field: - json - _temp ignore_missing: true - remove: + tag: remove_57c826fd if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - lastpass.event_report.time @@ -498,6 +560,7 @@ processors: ignore_failure: true ignore_missing: true - script: + tag: script_a3eb5add description: Drops null/empty values recursively. lang: painless source: | diff --git a/packages/lastpass/data_stream/user/elasticsearch/ingest_pipeline/default.yml b/packages/lastpass/data_stream/user/elasticsearch/ingest_pipeline/default.yml index 99875565fd2..bceb3d77b59 100644 --- a/packages/lastpass/data_stream/user/elasticsearch/ingest_pipeline/default.yml +++ b/packages/lastpass/data_stream/user/elasticsearch/ingest_pipeline/default.yml @@ -2,9 +2,11 @@ description: Pipeline for processing User logs. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -16,18 +18,23 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - set: + tag: set_event_kind_871e09cf field: event.kind value: state - set: + tag: set_event_category_ee497bfd field: event.category value: [iam] - set: + tag: set_event_type_1e7550ed field: event.type value: [user] - date: + tag: date_json_created_to_lastpass_user_created_142795a3 field: json.created target_field: lastpass.user.created formats: @@ -38,41 +45,51 @@ processors: if: ctx.json?.created != null && ctx.json.created != '' on_failure: - append: + tag: append_error_message_a4505198 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_id_to_lastpass_user_id_3ecd1792 field: json.id target_field: lastpass.user.id ignore_missing: true - set: + tag: set_user_id_8cbe6cb8 field: user.id copy_from: lastpass.user.id ignore_empty_value: true - rename: + tag: rename_json_username_to_lastpass_user_user_name_84381b37 field: json.username target_field: lastpass.user.user_name ignore_missing: true - set: + tag: set_user_email_d4cf99fd field: user.email copy_from: lastpass.user.user_name ignore_empty_value: true - rename: + tag: rename_json_fullname_to_lastpass_user_full_name_fd80e1c7 field: json.fullname target_field: lastpass.user.full_name ignore_missing: true - set: + tag: set_user_full_name_986324d0 field: user.full_name copy_from: lastpass.user.full_name ignore_empty_value: true - rename: + tag: rename_json_groups_to_lastpass_user_group_bf511ea7 field: json.groups target_field: lastpass.user.group ignore_missing: true - set: + tag: set_user_group_name_1d46c847 field: user.group.name copy_from: lastpass.user.group ignore_empty_value: true - convert: + tag: convert_json_applications_to_lastpass_user_application_884c6fa5 field: json.applications target_field: lastpass.user.application if: ctx.json?.applications != '' @@ -80,9 +97,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_41700d2a field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_attachments_to_lastpass_user_attachment_3d283ffc field: json.attachments target_field: lastpass.user.attachment if: ctx.json?.attachments != '' @@ -90,9 +109,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_73fb6dc1 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_disabled_to_lastpass_user_disabled_b13aba95 field: json.disabled target_field: lastpass.user.disabled if: ctx.json?.disabled != '' @@ -100,13 +121,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_58328b4e field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_duousername_to_lastpass_user_duo_user_name_f494576f field: json.duousername target_field: lastpass.user.duo.user_name ignore_missing: true - convert: + tag: convert_json_formfills_to_lastpass_user_form_fill_e8dde37f field: json.formfills target_field: lastpass.user.form_fill if: ctx.json?.formfills != '' @@ -114,9 +138,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e96e1e04 field: error.message value: '{{{_ingest.on_failure_message}}}' - date: + tag: date_json_last_login_to_lastpass_user_last_login_5b86a90f field: json.last_login target_field: lastpass.user.last_login formats: @@ -127,9 +153,11 @@ processors: if: ctx.json?.last_login != null && ctx.json.last_login != '' on_failure: - append: + tag: append_error_message_c5b017a0 field: error.message value: '{{{_ingest.on_failure_message}}}' - date: + tag: date_json_last_pw_change_to_lastpass_user_last_password_change_6dd9fdbb field: json.last_pw_change target_field: lastpass.user.last_password_change formats: @@ -140,13 +168,16 @@ processors: if: ctx.json?.last_pw_change != null && ctx.json.last_pw_change != '' on_failure: - append: + tag: append_error_message_3505bd80 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_linked_to_lastpass_user_linked_f90b3362 field: json.linked target_field: lastpass.user.linked ignore_missing: true - convert: + tag: convert_json_mpstrength_to_lastpass_user_master_password_strength_f38a4fab field: json.mpstrength target_field: lastpass.user.master_password_strength if: ctx.json?.mpstrength != '' @@ -154,9 +185,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d166e6ae field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_neverloggedin_to_lastpass_user_never_logged_in_a9aa2c92 field: json.neverloggedin target_field: lastpass.user.never_logged_in if: ctx.json?.neverloggedin != '' @@ -164,9 +197,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ec83f86f field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_notes_to_lastpass_user_note_30b2e6f9 field: json.notes target_field: lastpass.user.note if: ctx.json?.notes != '' @@ -174,9 +209,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e45556f8 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_password_reset_required_to_lastpass_user_password_reset_required_0534caf8 field: json.password_reset_required target_field: lastpass.user.password_reset_required if: ctx.json?.password_reset_required != '' @@ -184,13 +221,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_59601159 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_sites_to_lastpass_user_sites_daf7f770 field: json.sites target_field: lastpass.user.sites ignore_missing: true - convert: + tag: convert_json_totalscore_to_lastpass_user_total_score_87e2fdfb field: json.totalscore target_field: lastpass.user.total_score if: ctx.json?.totalscore != '' @@ -198,9 +238,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_1b093480 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_related_user_ba7b5c4b field: related.user value: - '{{{user.email}}}' @@ -208,10 +250,12 @@ processors: allow_duplicates: false ignore_failure: true - remove: + tag: remove_e5a80934 field: - json ignore_missing: true - remove: + tag: remove_815d31bb if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - lastpass.user.id @@ -221,6 +265,7 @@ processors: ignore_failure: true ignore_missing: true - script: + tag: script_e2ccbdd7 description: Drops null/empty values recursively. lang: painless source: | diff --git a/packages/lastpass/manifest.yml b/packages/lastpass/manifest.yml index 7f237062e1e..31108f1c001 100644 --- a/packages/lastpass/manifest.yml +++ b/packages/lastpass/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.3.2" name: lastpass title: LastPass -version: "1.23.2" +version: "1.23.3" description: Collect logs from LastPass with Elastic Agent. type: integration categories: diff --git a/packages/lumos/changelog.yml b/packages/lumos/changelog.yml index 228389e2348..9af54d9dbc1 100644 --- a/packages/lumos/changelog.yml +++ b/packages/lumos/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.8.3" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.8.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/lumos/data_stream/activity_logs/elasticsearch/ingest_pipeline/default.yml b/packages/lumos/data_stream/activity_logs/elasticsearch/ingest_pipeline/default.yml index dbc71e87f63..16e59396d8c 100644 --- a/packages/lumos/data_stream/activity_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/lumos/data_stream/activity_logs/elasticsearch/ingest_pipeline/default.yml @@ -2,42 +2,53 @@ description: Pipeline for processing Lumos Activity Logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: 8.11.0 - json: + tag: json_message_to_lumos_activity_logs_b4945de5 field: message target_field: lumos.activity_logs - rename: + tag: rename_lumos_activity_logs_event_hash_to_event_id_a4abd2af field: lumos.activity_logs.event_hash target_field: event.id ignore_missing: true - rename: + tag: rename_lumos_activity_logs_event_type_to_event_action_121254c6 field: lumos.activity_logs.event_type target_field: event.action ignore_missing: true - rename: + tag: rename_lumos_activity_logs_outcome_to_event_outcome_00c05161 field: lumos.activity_logs.outcome target_field: event.outcome ignore_missing: true - set: + tag: set_event_outcome_59b22e55 field: event.outcome value: unknown if: (ctx.event.outcome != "Failed") && (ctx.event.outcome != "Succeeded") - set: + tag: set_event_outcome_7fa84f25 field: event.outcome value: failure if: ctx.event.outcome == "Failed" - set: + tag: set_event_outcome_bade9e08 field: event.outcome value: success if: ctx.event.outcome == "Succeeded" - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] - script: + tag: script_9d6fa8ba description: Drops null/empty values recursively lang: painless ignore_failure: true diff --git a/packages/lumos/manifest.yml b/packages/lumos/manifest.yml index 6e3c2263dad..95205928593 100644 --- a/packages/lumos/manifest.yml +++ b/packages/lumos/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: lumos title: "Lumos" -version: "1.8.2" +version: "1.8.3" description: "An integration with Lumos to ship your Activity logs to your Elastic instance." type: integration categories: diff --git a/packages/lyve_cloud/changelog.yml b/packages/lyve_cloud/changelog.yml index c715dbcd3d4..b7cac37cb3a 100644 --- a/packages/lyve_cloud/changelog.yml +++ b/packages/lyve_cloud/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.17.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.17.1" changes: - description: Add `region` variable in the aws-s3 input to fix compatibility with Elastic Agent 8.19.12+ when using non-AWS S3-compatible buckets. diff --git a/packages/lyve_cloud/data_stream/audit/elasticsearch/ingest_pipeline/audit_lc.yml b/packages/lyve_cloud/data_stream/audit/elasticsearch/ingest_pipeline/audit_lc.yml index 2c6237e5992..1b46c7ce9da 100755 --- a/packages/lyve_cloud/data_stream/audit/elasticsearch/ingest_pipeline/audit_lc.yml +++ b/packages/lyve_cloud/data_stream/audit/elasticsearch/ingest_pipeline/audit_lc.yml @@ -3,36 +3,44 @@ description: Pipeline for Lyve Cloud audit logs processors: - json: + tag: json_event_original_to_lyve_cloud_audit_4d6ce823 field: event.original target_field: lyve_cloud.audit - rename: + tag: rename_lyve_cloud_audit_serviceAccountName_to_user_name_7de9218b field: lyve_cloud.audit.serviceAccountName target_field: user.name - rename: + tag: rename_lyve_cloud_audit_serviceAccountCreatorId_to_user_id_0d4d41af field: lyve_cloud.audit.serviceAccountCreatorId target_field: user.id - set: + tag: set_user_email_10d967c0 copy_from: user.id field: user.email - append: + tag: append_related_user_e96ba408 field: related.user value: "{{{user.name}}}" allow_duplicates: false if: ctx.user?.name != null && ctx.user.name != "" - append: + tag: append_related_user_5aa8868e field: related.user value: "{{{user.id}}}" allow_duplicates: false if: ctx.user?.name != null && ctx.user.name != "" - set: + tag: set_cloud_provider_253b5155 if: ctx?.log?.file?.path != null && ctx.log.file.path.contains("seagate.com") field: cloud.provider value: "lyvecloud" - date: + tag: date_lyve_cloud_audit_auditEntry_time_4f513afb field: lyve_cloud.audit.auditEntry.time formats: - "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'" @@ -46,60 +54,74 @@ processors: - "yyyy-MM-dd'T'HH:mm:ss.S'Z'" - "yyyy-MM-dd'T'HH:mm:ss'Z'" - user_agent: + tag: user_agent_lyve_cloud_audit_auditEntry_userAgent_df5e2d5c field: lyve_cloud.audit.auditEntry.userAgent - rename: + tag: rename_lyve_cloud_audit_auditEntry_api_statusCode_to_http_response_status_code_b3f5e86d field: lyve_cloud.audit.auditEntry.api.statusCode target_field: http.response.status_code - gsub: + tag: gsub_lyve_cloud_audit_auditEntry_api_timeToFirstByte_19318e8c field: lyve_cloud.audit.auditEntry.api.timeToFirstByte pattern: "ns" replacement: "" ignore_missing: true - gsub: + tag: gsub_lyve_cloud_audit_auditEntry_api_timeToResponse_63f2101e field: lyve_cloud.audit.auditEntry.api.timeToResponse pattern: "ns" replacement: "" - - convert: - field: lyve_cloud.audit.auditEntry.api.timeToFirstByte + - convert: + tag: convert_lyve_cloud_audit_auditEntry_api_timeToFirstByte_ee05a3d3 + field: lyve_cloud.audit.auditEntry.api.timeToFirstByte type: long ignore_missing: true - - convert: - field: lyve_cloud.audit.auditEntry.api.timeToResponse + - convert: + tag: convert_lyve_cloud_audit_auditEntry_api_timeToResponse_0099ba9e + field: lyve_cloud.audit.auditEntry.api.timeToResponse type: long ignore_missing: true - - convert: + - convert: + tag: convert_lyve_cloud_audit_auditEntry_responseHeader_Content-Length_52614149 field: lyve_cloud.audit.auditEntry.responseHeader.Content-Length type: long ignore_missing: true - - convert: + - convert: + tag: convert_lyve_cloud_audit_auditEntry_requestHeader_Content-Length_6b625753 field: lyve_cloud.audit.auditEntry.requestHeader.Content-Length type: long ignore_missing: true - rename: + tag: rename_lyve_cloud_audit_auditEntry_requestHeader_Content-Length_to_http_request_body_bytes_c68abc22 field: lyve_cloud.audit.auditEntry.requestHeader.Content-Length target_field: http.request.body.bytes ignore_missing: true - rename: + tag: rename_lyve_cloud_audit_auditEntry_responseHeader_Content-Length_to_http_response_body_bytes_e0efb0d8 field: lyve_cloud.audit.auditEntry.responseHeader.Content-Length target_field: http.response.body.bytes ignore_missing: true - - rename: + - rename: + tag: rename_lyve_cloud_audit_auditEntry_responseHeader_Content-Type_to_http_response_mime_type_704589d6 field: lyve_cloud.audit.auditEntry.responseHeader.Content-Type target_field: http.response.mime_type ignore_missing: true - - convert: + - convert: + tag: convert_lyve_cloud_audit_auditEntry_requestHeader_X-Forwarded-Port_319c5d42 field: lyve_cloud.audit.auditEntry.requestHeader.X-Forwarded-Port type: long - ignore_missing: true + ignore_missing: true - rename: + tag: rename_lyve_cloud_audit_auditEntry_responseHeader_X-Amz-Object-Lock-Retain-Until-Date_to_lyve_cloud_audit_auditEntry_responseHeader_object_lock_retain_until_date_bf96801c field: lyve_cloud.audit.auditEntry.responseHeader.X-Amz-Object-Lock-Retain-Until-Date target_field: lyve_cloud.audit.auditEntry.responseHeader.object_lock_retain_until_date ignore_missing: true - date: - if: ctx?.lyve_cloud?.audit?.auditEntry?.responseHeader?.ObjectLockRetainUntilDate != null && ctx.lyve_cloud.audit.auditEntry.responseHeader.ObjectLockRetainUntilDate!="" + tag: date_lyve_cloud_audit_auditEntry_responseHeader_ObjectLockRetainUntilDate_to_lyve_cloud_audit_auditEntry_responseHeader_ObjectLockRetainUntilDate_092a19d3 + if: ctx?.lyve_cloud?.audit?.auditEntry?.responseHeader?.ObjectLockRetainUntilDate != null && ctx.lyve_cloud.audit.auditEntry.responseHeader.ObjectLockRetainUntilDate!="" field: lyve_cloud.audit.auditEntry.responseHeader.ObjectLockRetainUntilDate formats: - "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'" @@ -115,23 +137,27 @@ processors: target_field: lyve_cloud.audit.auditEntry.responseHeader.ObjectLockRetainUntilDate - append: + tag: append_related_hosts_40160b64 field: related.hosts value: "{{lyve_cloud.audit.auditEntry.requestHeader.X-Forwarded-Host}}" allow_duplicates: false if: ctx.lyve_cloud?.audit?.auditEntry?.requestHeader != null && ctx.lyve_cloud.audit.auditEntry.requestHeader["X-Forwarded-Host"] != "" - split: + tag: split_lyve_cloud_audit_auditEntry_requestHeader_X-Forwarded-For_to_related_ip_793fd8c0 field: lyve_cloud.audit.auditEntry.requestHeader.X-Forwarded-For target_field: related.ip separator: "," ignore_missing: true - foreach: + tag: foreach_related_ip_59641ceb field: related.ip processor: trim: field: "_ingest._value" ignore_missing: true - foreach: + tag: foreach_related_ip_4ad20826 description: Ensure related.ip type field: related.ip processor: @@ -144,11 +170,13 @@ processors: value: "{{{_ingest._value}}}" ignore_missing: true - script: + tag: script_b2fcd558 lang: painless source: ctx.related.ip.removeAll(ctx["_failed_ips"]); if: ctx.related?.ip != null && ctx["_failed_ips"] != null; - script: + tag: script_bf705a2a description: "setting client's ip as the last element of the ' related.ip ' field ( the rest are proxies)" lang: painless source: >- @@ -157,10 +185,12 @@ processors: if: ctx.related?.ip != null && ctx.related.ip.length > 0 - geoip: + tag: geoip_client_ip_to_client_geo_90831252 field: client.ip target_field: client.geo ignore_missing: true - geoip: + tag: geoip_client_ip_to_client_as_f17fb2b3 database_file: GeoLite2-ASN.mmdb field: client.ip target_field: client.as @@ -169,20 +199,24 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_client_as_asn_to_client_as_number_a6e30d01 field: client.as.asn target_field: client.as.number ignore_missing: true - rename: + tag: rename_client_as_organization_name_to_client_as_organization_name_817a526f field: client.as.organization_name target_field: client.as.organization.name ignore_missing: true - set: + tag: set_source_d6ee6995 field: source copy_from: client if: ctx.client != null - remove: - field: + tag: remove_a37911c1 + field: - lyve_cloud.audit.auditEntry.time - lyve_cloud.audit.auditEntry.userAgent - lyve_cloud.audit.auditEntry.requestHeader.User-Agent @@ -215,3 +249,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/lyve_cloud/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/lyve_cloud/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index dfaf7a1a214..008dc6034a6 100644 --- a/packages/lyve_cloud/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/lyve_cloud/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -2,21 +2,26 @@ description: Pipeline for Lyve Cloud processors: - drop: + tag: drop_e829390e description: Drop any logs that are not audit. if: ctx?.log?.file?.path == null || !ctx.log.file.path.contains("/S3") - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - rename: + tag: rename_message_to_event_original_56a77271 field: "message" target_field: "event.original" ignore_missing: true if: ctx.event?.original == null - pipeline: + tag: pipeline_cc3554b9 description: Sends audit logs to their pipeline. name: '{{ IngestPipeline "audit_lc" }}' if: ctx?.log?.file?.path != null && ctx.log.file.path.contains("/S3") - fingerprint: + tag: fingerprint_d8228bd1 description: Create a custom document id for deduplication. target_field: _id fields: diff --git a/packages/lyve_cloud/manifest.yml b/packages/lyve_cloud/manifest.yml index 02b4041b6ac..c29b82dc516 100644 --- a/packages/lyve_cloud/manifest.yml +++ b/packages/lyve_cloud/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: lyve_cloud title: Lyve Cloud -version: "1.17.1" +version: "1.17.2" description: Collect S3 API audit log from Lyve Cloud with Elastic Agent. type: integration categories: diff --git a/packages/m365_defender/changelog.yml b/packages/m365_defender/changelog.yml index bd6bf924cb7..fea997691e3 100644 --- a/packages/m365_defender/changelog.yml +++ b/packages/m365_defender/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "5.15.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "5.15.1" changes: - description: Fix `alert` and `incident` pagination to advance the `$filter` time boundary (`lastUpdateDateTime gt `) with `$skip` held at 0, instead of incrementing `$skip`. This avoids the Graph security API `$skip <= 500` limit (which caused `400` errors and stalled catch-up under high volume) and drains a backlog in a single interval. Also correct the `alert` `batch_size` default from 2000 to 1000, matching the API `$top` cap. diff --git a/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_alert.yml b/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_alert.yml index 194b4535587..55e0b941b64 100644 --- a/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_alert.yml +++ b/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_alert.yml @@ -888,3 +888,7 @@ on_failure: - set: field: event.kind value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_app_and_identity.yml b/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_app_and_identity.yml index 8c26f68b1c3..f3dba052530 100644 --- a/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_app_and_identity.yml +++ b/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_app_and_identity.yml @@ -1286,3 +1286,7 @@ on_failure: - set: field: event.kind value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_device.yml b/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_device.yml index cd2d22860c7..d3f19871ac9 100644 --- a/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_device.yml +++ b/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_device.yml @@ -3194,3 +3194,7 @@ on_failure: - set: field: event.kind value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_email.yml b/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_email.yml index 94652cac836..a5d6d4846ac 100644 --- a/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_email.yml +++ b/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_email.yml @@ -766,3 +766,7 @@ on_failure: - set: field: event.kind value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/m365_defender/manifest.yml b/packages/m365_defender/manifest.yml index dcefff3fedd..e57abc9a28a 100644 --- a/packages/m365_defender/manifest.yml +++ b/packages/m365_defender/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.4.0" name: m365_defender title: Microsoft Defender XDR -version: "5.15.1" +version: "5.15.2" description: Collect logs from Microsoft Defender XDR with Elastic Agent. categories: - "security" diff --git a/packages/mattermost/changelog.yml b/packages/mattermost/changelog.yml index d714516e0cb..2a324f85d44 100644 --- a/packages/mattermost/changelog.yml +++ b/packages/mattermost/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.5.1" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "2.5.0" changes: - description: Add dashboard. diff --git a/packages/mattermost/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/mattermost/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 4dc96fee4f1..85b4ae1e1f4 100644 --- a/packages/mattermost/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mattermost/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -2,88 +2,108 @@ description: Pipeline for processing Mattermost audit logs processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - fail: + tag: fail_74459f8f if: "!(ctx.json instanceof Map)" message: Missing JSON object - date: + tag: date_json_timestamp_to_timestamp_c024a8df field: json.timestamp formats: - yyyy-MM-dd HH:mm:ss.SSS 'Z' timezone: UTC target_field: "@timestamp" - rename: + tag: rename_json_event_to_event_action_947fc404 field: json.event target_field: event.action ignore_missing: true - rename: + tag: rename_json_err_to_error_code_6b1447e6 field: json.err target_field: error.code ignore_missing: true - rename: + tag: rename_json_errors_to_mattermost_audit_error_message_b6f15f90 field: json.errors target_field: mattermost.audit.error.message ignore_missing: true if: ctx.json.errors != "[]" - gsub: + tag: gsub_mattermost_audit_error_message_317838ce field: mattermost.audit.error.message pattern: "(\\[|\\])" replacement: "" ignore_missing: true - split: + tag: split_mattermost_audit_error_message_49aecd34 field: mattermost.audit.error.message separator: ",\\s+" ignore_missing: true ignore_failure: true - set: + tag: set_event_outcome_20f1cdfe field: event.outcome value: success if: ctx.json.status == "success" - set: + tag: set_event_outcome_091b384f field: event.outcome value: failure if: ctx.json.status == "fail" || ctx.mattermost?.audit?.error?.message != null - set: + tag: set_event_outcome_5ffb44a5 field: event.outcome value: unknown if: ctx.event.outcome == null - rename: + tag: rename_json_user_id_to_user_id_4eee742f field: json.user_id target_field: user.id ignore_missing: true - rename: + tag: rename_json_user_id_to_user_id_4eee742f field: json.user_id target_field: user.id ignore_missing: true - rename: + tag: rename_json_login_id_to_user_id_80c33d61 field: json.login_id target_field: user.id ignore_missing: true if: ctx.user?.id == null - rename: + tag: rename_json_ip_address_to_source_address_1490d4b5 field: json.ip_address target_field: source.address ignore_missing: true - convert: + tag: convert_source_address_to_source_ip_f5632a20 field: source.address target_field: source.ip type: ip ignore_missing: true ignore_failure: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -92,99 +112,123 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - user_agent: + tag: user_agent_json_client_to_user_agent_c81d6fef field: json.client target_field: user_agent ignore_missing: true - rename: + tag: rename_json_api_path_to_mattermost_audit_api_path_eecf6eef field: json.api_path target_field: mattermost.audit.api_path ignore_missing: true - uri_parts: + tag: uri_parts_mattermost_audit_api_path_2a14bf06 field: mattermost.audit.api_path ignore_failure: true - rename: + tag: rename_json_session_id_to_mattermost_audit_session_id_a07c2c76 field: json.session_id target_field: mattermost.audit.session.id ignore_missing: true - rename: + tag: rename_json_device_id_to_mattermost_audit_device_id_9fa3394e field: json.device_id target_field: mattermost.audit.device.id ignore_missing: true - rename: + tag: rename_json_cluster_id_to_mattermost_audit_cluster_id_2b6d299a field: json.cluster_id target_field: mattermost.audit.cluster.id ignore_missing: true - rename: + tag: rename_json_user_id_to_user_target_id_391b4f99 field: json.user.id target_field: user.target.id ignore_missing: true - rename: + tag: rename_json_user_name_to_user_target_name_2e89dc61 field: json.user.name target_field: user.target.name ignore_missing: true - rename: + tag: rename_json_user_roles_to_user_target_roles_9ee510c1 field: json.user.roles target_field: user.target.roles ignore_missing: true - split: + tag: split_user_target_roles_411917ca field: user.target.roles separator: \s+ ignore_missing: true - rename: + tag: rename_json_remove_user_id_to_user_target_id_9efc768b field: json.remove_user_id target_field: user.target.id ignore_missing: true - gsub: + tag: gsub_json_user_ids_7142ff32 field: json.user_ids pattern: "(\\[|\\])" replacement: "" ignore_missing: true - split: + tag: split_json_user_ids_00c0dc92 field: json.user_ids separator: \s+ ignore_missing: true ignore_failure: true - rename: + tag: rename_json_user_ids_to_user_target_id_157040a3 field: json.user_ids target_field: user.target.id ignore_missing: true - rename: + tag: rename_json_team_to_mattermost_audit_team_c9209f35 field: json.team target_field: mattermost.audit.team ignore_missing: true - rename: + tag: rename_json_code_to_http_response_status_code_b30a39c8 field: json.code target_field: http.response.status_code ignore_missing: true - rename: + tag: rename_json_post_to_mattermost_audit_post_ebb40d6f field: json.post target_field: mattermost.audit.post ignore_missing: true - rename: + tag: rename_mattermost_audit_post_channel_id_to_mattermost_audit_post_channel_id_a48e1e69 field: mattermost.audit.post.channel_id target_field: mattermost.audit.post.channel.id ignore_missing: true - rename: + tag: rename_json_patch_to_mattermost_audit_patch_8500553b field: json.patch target_field: mattermost.audit.patch ignore_missing: true - rename: + tag: rename_json_patched_to_mattermost_audit_patch_1e007f18 field: json.patched target_field: mattermost.audit.patch ignore_missing: true - rename: + tag: rename_json_channel_to_mattermost_audit_channel_d79aa57f field: json.channel target_field: mattermost.audit.channel ignore_missing: true - rename: + tag: rename_json_channeld_to_mattermost_audit_channel_13b89b9f field: json.channeld target_field: mattermost.audit.channel ignore_missing: true @@ -321,6 +365,7 @@ processors: def hm = new HashMap(params.get(ctx.event.action)); hm.forEach((k, v) -> ctx.event[k] = v); - script: + tag: script_b86d15c2 lang: painless description: Add ECS User fields if: ctx.event.category.contains('iam') @@ -360,26 +405,31 @@ processors: ctx.user.target.group.put("id", ctx.mattermost?.audit?.team?.id); } - append: + tag: append_related_user_837e080f field: related.user value: '{{{user.name}}}' allow_duplicates: false if: ctx.user?.name != null - append: + tag: append_related_user_d2acb44a field: related.user value: '{{{user.changes.name}}}' allow_duplicates: false if: ctx.user?.changes?.name != null - append: + tag: append_related_user_7f407fef field: related.user value: '{{{user.id}}}' allow_duplicates: false if: ctx.user?.id != null - append: + tag: append_related_user_0b822a80 field: related.user value: '{{{user.target.id}}}' allow_duplicates: false if: ctx.user?.target?.id instanceof String - foreach: + tag: foreach_user_target_id_dcd3a36b field: user.target.id processor: append: @@ -389,30 +439,36 @@ processors: ignore_missing: true if: ctx.user?.target?.id instanceof List - append: + tag: append_related_ip_8121c591 field: related.ip value: '{{{source.ip}}}' allow_duplicates: false if: ctx.source?.ip != null - append: + tag: append_mattermost_audit_related_channel_abecccbd field: mattermost.audit.related.channel value: '{{{mattermost.audit.post.channel.id}}}' allow_duplicates: false if: ctx.mattermost?.audit?.post?.channel?.id != null - append: + tag: append_mattermost_audit_related_channel_1bdbcbaa field: mattermost.audit.related.channel value: '{{{mattermost.audit.channel.id}}}' allow_duplicates: false if: ctx.mattermost?.audit?.channel?.id != null - append: + tag: append_mattermost_audit_related_team_8fed3d9e field: mattermost.audit.related.team value: '{{{mattermost.audit.team.id}}}' allow_duplicates: false if: ctx.mattermost?.audit?.team?.id != null - remove: + tag: remove_e5a80934 field: - json ignore_missing: true - script: + tag: script_4d6993cb lang: painless description: This script processor iterates over the whole document to remove fields with null values. source: | diff --git a/packages/mattermost/manifest.yml b/packages/mattermost/manifest.yml index c14289ca98f..8032b5a8c2d 100644 --- a/packages/mattermost/manifest.yml +++ b/packages/mattermost/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: mattermost title: "Mattermost" -version: "2.5.0" +version: "2.5.1" description: Collect logs from Mattermost with Elastic Agent. type: integration categories: diff --git a/packages/menlo/changelog.yml b/packages/menlo/changelog.yml index 0eb0a29032a..fb5b4d9145d 100644 --- a/packages/menlo/changelog.yml +++ b/packages/menlo/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.8.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.8.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/menlo/data_stream/dlp/elasticsearch/ingest_pipeline/default.yml b/packages/menlo/data_stream/dlp/elasticsearch/ingest_pipeline/default.yml index 2ad369536a7..806971af290 100644 --- a/packages/menlo/data_stream/dlp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/menlo/data_stream/dlp/elasticsearch/ingest_pipeline/default.yml @@ -2,6 +2,7 @@ description: Mapping of Menlo DLP Fields from API into ECS fields. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: 8.11.0 - rename: @@ -17,6 +18,7 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - set: + tag: set_event_kind_39295792 field: event.kind value: alert - json: @@ -24,6 +26,7 @@ processors: tag: 'json_decoding' target_field: json - rename: + tag: rename_json_event_event_id_to_event_id_f7428aad field: json.event.event_id target_field: event.id ignore_missing: true @@ -36,84 +39,105 @@ processors: tag: date_event_created_time_epoch on_failure: - append: + tag: append_error_message_21aa8fb4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - uri_parts: + tag: uri_parts_json_event_dst_url_to_url_599c955a field: json.event.dst_url target_field: "url" keep_original: true remove_if_successful: true if: ctx.json?.event?.dst_url != null - registered_domain: + tag: registered_domain_url_domain_to_url_5d82631c field: url.domain target_field: url - rename: + tag: rename_json_event_rule_id_to_rule_id_d1b284f7 field: json.event.rule_id target_field: rule.id ignore_missing: true - rename: + tag: rename_json_event_action_to_event_action_3087f43a field: json.event.action target_field: event.action ignore_missing: true - set: + tag: set_event_action_aefac6e9 field: event.action value: blocked if: ctx.json.event.action == 'block' - set: + tag: set_event_action_e3218850 field: event.action value: log if: ctx.json.event.action == 'log' - set: + tag: set_event_type_40ce1cb9 field: event.type value: denied if: ctx.event.action == 'blocked' - rename: + tag: rename_json_event_rule_name_to_rule_name_fd2c7463 field: json.event.rule_name target_field: rule.name ignore_missing: true - set: + tag: set_event_category_6853894d field: event.category value: intrusion_detection - append: + tag: append_event_category_7afdca3c field: event.category value: network - convert: + tag: convert_json_event_severity_to_event_severity_e3a83e4f field: "json.event.severity" type: long target_field: "event.severity" ignore_missing: true - set: + tag: set_event_outcome_82c7155e field: event.outcome value: unknown - set: + tag: set_event_outcome_d720f751 field: event.outcome value: success if: ctx.event.action == 'log' || ctx.event.action == 'block' - rename: + tag: rename_json_event_product_to_observer_product_d119c06e field: json.event.product target_field: observer.product ignore_missing: true - rename: + tag: rename_json_event_vendor_to_observer_vendor_df71a640 field: json.event.vendor target_field: observer.vendor ignore_missing: true - rename: + tag: rename_json_event_version_to_observer_version_ee7a1daa field: json.event.version target_field: observer.version ignore_missing: true - rename: + tag: rename_json_event_request_type_to_http_request_method_3d7bf26c field: json.event.request_type target_field: http.request.method ignore_missing: true - rename: + tag: rename_menlo_protocol_to_network_protocol_719e035f field: menlo.protocol target_field: network.protocol ignore_missing: true - rename: + tag: rename_json_event_userid_to_user_id_f5aee914 field: json.event.userid target_field: user.id ignore_missing: true - grok: + tag: grok_user_id_893ebf56 field: user.id patterns: - '%{USERNAME:user.name}@%{HOSTNAME:user.domain}' @@ -126,60 +150,73 @@ processors: tag: copy_user_email if: ctx.user?.id != null && ctx.user.id.indexOf("@") > 0 - append: + tag: append_related_user_837e080f field: related.user value: '{{{user.name}}}' allow_duplicates: false if: ctx.user?.name != null - append: + tag: append_related_user_34fcf415 field: related.user value: '{{{user.email}}}' allow_duplicates: false if: ctx.user?.email != null - rename: + tag: rename_json_event_filename_to_file_name_f289d164 field: json.event.filename target_field: file.name ignore_missing: true - rename: + tag: rename_json_event_sha256_to_file_hash_sha256_9edd9b99 field: json.event.sha256 target_field: file.hash.sha256 if: ctx.json?.sha256 != "NA" ignore_missing: true - append: + tag: append_related_hash_0ce6a892 field: related.hash value: "{{{file.hash.sha256}}}" allow_duplicates: false - if: ctx.file?.hash.sha256 != null + if: ctx.file?.hash.sha256 != null - rename: + tag: rename_json_event_categories_to_menlo_dlp_category_5879914d field: json.event.categories target_field: menlo.dlp.category ignore_missing: true - convert: + tag: convert_json_event_ccl_match_counts_to_menlo_dlp_ccl_match_counts_26c172bc ignore_failure: true field: json.event.ccl_match_counts type: long target_field: menlo.dlp.ccl.match_counts ignore_missing: true - rename: + tag: rename_json_event_user_input_to_menlo_dlp_user_input_b3161409 field: json.event.user_input target_field: menlo.dlp.user_input ignore_missing: true - rename: + tag: rename_json_event_alerted_to_menlo_dlp_alerted_9b26efff field: json.event.alerted target_field: menlo.dlp.alerted ignore_missing: true - rename: + tag: rename_json_event_status_to_menlo_dlp_status_bed925cd field: json.event.status target_field: menlo.dlp.status ignore_missing: true - convert: + tag: convert_json_event_ccl_scores_to_menlo_dlp_ccl_score_0460fcce field: json.event.ccl_scores type: long target_field: menlo.dlp.ccl.score ignore_missing: true - rename: + tag: rename_json_event_ccl_ids_to_menlo_dlp_ccl_id_6447cb06 field: json.event.ccl_ids target_field: menlo.dlp.ccl.id - rename: + tag: rename_json_event_stream_name_to_menlo_dlp_stream_name_5d456840 field: json.event.stream_name target_field: menlo.dlp.stream_name - remove: @@ -205,11 +242,13 @@ processors: } drop(ctx); - append: + tag: append_event_kind_d2f25e35 field: event.kind value: pipeline_error allow_duplicates: false if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/menlo/data_stream/web/elasticsearch/ingest_pipeline/default.yml b/packages/menlo/data_stream/web/elasticsearch/ingest_pipeline/default.yml index d6347e2b129..a10e1f20f75 100644 --- a/packages/menlo/data_stream/web/elasticsearch/ingest_pipeline/default.yml +++ b/packages/menlo/data_stream/web/elasticsearch/ingest_pipeline/default.yml @@ -2,6 +2,7 @@ description: Mapping of Menlo Web Proxy Fields from API into ECS fields. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: 8.11.0 - rename: @@ -17,6 +18,7 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - set: + tag: set_event_kind_39295792 field: event.kind value: alert - json: @@ -32,168 +34,208 @@ processors: tag: date_timestamp_epoch on_failure: - append: + tag: append_error_message_a65d3437 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_event_severity_to_event_severity_e3a83e4f field: json.event.severity type: long target_field: event.severity ignore_missing: true - rename: + tag: rename_json_event_full_session_id_to_event_id_88f11471 field: json.event.full_session_id target_field: event.id ignore_missing: true - rename: + tag: rename_json_event_pe_action_to_event_action_a00661dc field: json.event.pe_action target_field: event.action ignore_missing: true - set: + tag: set_event_outcome_82c7155e field: event.outcome value: unknown - set: + tag: set_event_outcome_176b2308 field: event.outcome value: failure if: ctx.event.action == 'block' || ctx.event.action == 'isolate' || ctx.event.action == 'ssl_exception' - set: + tag: set_event_outcome_0598fac0 field: event.outcome value: success if: ctx.event.action == 'direct' || ctx.event.action == 'allow' - rename: + tag: rename_json_event_pe_reason_to_event_reason_4ae02444 field: json.event.pe_reason target_field: event.reason ignore_missing: true - set: + tag: set_event_category_7a832770 field: event.category value: web - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_category_4cc22c32 field: event.category value: threat - rename: + tag: rename_json_event_name_to_menlo_web_request_type_4c7adffc field: json.event.name target_field: menlo.web.request_type ignore_missing: true - rename: + tag: rename_json_event_soph_dlp_ref_to_event_reference_fff80ba8 field: json.event.soph_dlp_ref target_field: event.reference if: ctx.json?.event?.soph_dlp_ref != "NA" ignore_missing: true - rename: + tag: rename_json_event_risk_score_to_menlo_web_risk_score_a6d3f05f field: json.event.risk_score target_field: menlo.web.risk_score ignore_missing: true - append: + tag: append_dns_answers_data_190583a6 field: dns.answers.data value: '{{{json.event.dst}}}' allow_duplicates: false - rename: + tag: rename_json_event_product_to_observer_product_d119c06e field: json.event.product target_field: observer.product ignore_missing: true - rename: + tag: rename_json_event_vendor_to_observer_vendor_df71a640 field: json.event.vendor target_field: observer.vendor ignore_missing: true - convert: + tag: convert_json_event_risk_tally_to_menlo_web_tally_0ef18826 field: json.event.risk_tally type: long target_field: menlo.web.tally ignore_missing: true - convert: + tag: convert_json_event_has_password_to_menlo_web_has_password_a6b13894 field: json.event.has_password type: boolean target_field: menlo.web.has_password if: ctx?.json?.event?.has_password != "NA" ignore_missing: true - convert: + tag: convert_json_event_x-client-ip_to_client_ip_e4296e31 field: json.event.x-client-ip type: ip target_field: client.ip ignore_missing: true ignore_failure: true - convert: + tag: convert_json_event_origin_ip_to_server_ip_f8a7c5a4 field: json.event.origin_ip type: ip target_field: server.ip ignore_failure: true - rename: + tag: rename_json_event_origin_country_to_server_geo_country_iso_code_b39bb1c3 field: json.event.origin_country target_field: server.geo.country_iso_code ignore_failure: true - rename: + tag: rename_json_event_x-client-country_to_client_geo_country_iso_code_2f0b9507 field: json.event.x-client-country target_field: client.geo.country_iso_code ignore_failure: true - rename: + tag: rename_json_event_egress_country_to_observer_geo_country_iso_code_2f689226 field: json.event.egress_country target_field: observer.geo.country_iso_code ignore_missing: true ignore_failure: true - convert: + tag: convert_json_event_file_size_to_file_size_6d8d4d21 field: json.event.file_size type: long target_field: file.size if: ctx.json?.event?.file_size != "NA" ignore_missing: true - append: + tag: append_observer_ip_fee8f961 field: observer.ip value: '{{{json.event.egress_ip}}}' allow_duplicates: false if: ctx.json?.event?.egress_ip != "NA" - set: + tag: set_destination_bb802684 field: destination copy_from: server ignore_empty_value: true - set: + tag: set_source_9aecde9f field: source copy_from: client - rename: + tag: rename_json_event_content-type_to_menlo_web_content_type_449b5225 field: json.event.content-type target_field: menlo.web.content_type ignore_missing: true - user_agent: + tag: user_agent_json_event_user-agent_9ffed892 field: json.event.user-agent ignore_missing: true - rename: + tag: rename_json_event_request_type_to_http_request_method_3d7bf26c field: json.event.request_type target_field: http.request.method ignore_missing: true - rename: + tag: rename_json_event_ua_type_to_menlo_web_ua_type_f8dd36f9 field: json.event.ua_type target_field: menlo.web.ua_type ignore_missing: true - rename: + tag: rename_json_event_version_to_observer_version_ee7a1daa field: json.event.version target_field: observer.version ignore_missing: true - uri_parts: + tag: uri_parts_json_event_url_c0d815a4 field: json.event.url keep_original: true remove_if_successful: true ignore_failure: true - registered_domain: + tag: registered_domain_url_domain_to_url_5d82631c field: url.domain target_field: url - rename: + tag: rename_json_event_referer_to_http_request_referrer_2243fb3f field: json.event.referer target_field: http.request.referrer ignore_missing: true - rename: + tag: rename_json_event_userid_to_user_name_dc78ba9a field: json.event.userid target_field: user.name ignore_missing: true - append: + tag: append_related_user_837e080f field: related.user value: "{{{user.name}}}" allow_duplicates: false - if: ctx.user?.name != null + if: ctx.user?.name != null - rename: + tag: rename_json_event_protocol_to_network_protocol_81f122d4 field: json.event.protocol target_field: network.protocol ignore_missing: true - convert: + tag: convert_json_event_response_code_to_http_response_status_code_679cc84d field: json.event.response_code type: long target_field: http.response.status_code @@ -204,126 +246,156 @@ processors: field: cloud.region ignore_empty_value: true - rename: + tag: rename_json_event_filename_to_file_name_482c1acc field: json.event.filename target_field: file.name ignore_missing: true if: ctx.json?.event?.filename != "NA" - rename: + tag: rename_json_event_sha256_to_file_hash_sha256_916bbda2 field: json.event.sha256 target_field: file.hash.sha256 ignore_missing: true - append: + tag: append_related_hash_017aef51 field: related.hash value: "{{{file.hash.sha256}}}" allow_duplicates: false - if: ctx.file?.hash?.sha256 != null + if: ctx.file?.hash?.sha256 != null - rename: + tag: rename_json_event_categories_to_menlo_web_categories_f756c68b field: json.event.categories target_field: menlo.web.categories ignore_missing: true - rename: + tag: rename_json_event_threats_to_menlo_web_threats_0734c689 field: json.event.threats target_field: menlo.web.threats ignore_missing: true - rename: + tag: rename_json_event_threat_types_to_menlo_web_threat_types_96f979e3 field: json.event.threat_types target_field: menlo.web.threat_types ignore_missing: true - rename: + tag: rename_json_event_is_iframe_to_menlo_web_is_iframe_8fe49fe1 field: json.event.is_iframe target_field: menlo.web.is_iframe ignore_missing: true - rename: + tag: rename_json_event_product_to_observer_product_d119c06e field: json.event.product target_field: observer.product ignore_missing: true - rename: + tag: rename_json_event_protocol_to_network_protocol_81f122d4 field: json.event.protocol target_field: network.protocol ignore_missing: true - rename: + tag: rename_json_event_cached_to_menlo_web_cached_4a8a0c37 field: json.event.cached target_field: menlo.web.cached ignore_missing: true - rename: + tag: rename_json_event_casb_app_name_to_menlo_web_casb_app_name_8f422a81 field: json.event.casb_app_name target_field: menlo.web.casb_app_name ignore_missing: true - rename: + tag: rename_json_event_casb_cat_name_to_menlo_web_casb_cat_name_6ebe0d97 field: json.event.casb_cat_name target_field: menlo.web.casb_cat_name ignore_missing: true - rename: + tag: rename_json_event_casb_fun_name_to_menlo_web_casb_fun_name_89227c1d field: json.event.casb_fun_name target_field: menlo.web.casb_fun_name ignore_missing: true - rename: + tag: rename_json_event_casb_org_name_to_menlo_web_casb_org_name_bcab44d3 field: json.event.casb_org_name target_field: menlo.web.casb_org_name ignore_missing: true - rename: + tag: rename_json_event_casb_profile_id_to_menlo_web_casb_profile_id_ef8363d1 field: json.event.casb_profile_id target_field: menlo.web.casb_profile_id ignore_missing: true - rename: + tag: rename_json_event_casb_profile_name_to_menlo_web_casb_profile_name_4268d8c5 field: json.event.casb_profile_name target_field: menlo.web.casb_profile_name ignore_missing: true - rename: + tag: rename_json_event_casb_profile_type_to_menlo_web_casb_profile_type_6d7c4573 field: json.event.casb_profile_type target_field: menlo.web.casb_profile_type ignore_missing: true - rename: + tag: rename_json_event_casb_risk_score_to_menlo_web_casb_risk_score_60773e53 field: json.event.casb_risk_score target_field: menlo.web.casb_risk_score ignore_missing: true - rename: + tag: rename_json_event_connId_to_menlo_web_conn_id_08d02118 field: json.event.connId target_field: menlo.web.conn_id ignore_missing: true - rename: + tag: rename_json_event_reqId_to_menlo_web_req_id_c7080fd6 field: json.event.reqId target_field: menlo.web.req_id ignore_missing: true - rename: + tag: rename_json_event_proxyEventType_to_menlo_web_proxy_event_type_d0b219fb field: json.event.proxyEventType target_field: menlo.web.proxy_event_type ignore_missing: true - rename: + tag: rename_json_event_proxyEventDetail_to_menlo_web_proxy_event_detail_675b199f field: json.event.proxyEventDetail target_field: menlo.web.proxy_event_detail ignore_missing: true - rename: + tag: rename_json_event_sbox_to_menlo_web_sbox_36cd1413 field: json.event.sbox target_field: menlo.web.sbox ignore_missing: true - rename: + tag: rename_json_event_sbox_mal_act_to_menlo_web_sbox_mal_act_e0597b03 field: json.event.sbox_mal_act target_field: menlo.web.sbox_mal_act ignore_missing: true - rename: + tag: rename_json_event_soph_to_menlo_web_soph_694b78d7 field: json.event.soph target_field: menlo.web.soph ignore_missing: true - rename: + tag: rename_json_event_tab_id_to_menlo_web_tab_id_6afd2bef field: json.event.tab_id target_field: menlo.web.tab_id ignore_missing: true - rename: + tag: rename_json_event_virus_details_to_menlo_web_virus_details_d0c58747 field: json.event.virus_details target_field: menlo.web.virus_details ignore_missing: true - rename: + tag: rename_json_event_xff_ip_to_menlo_web_xff_ip_1f9322db field: json.event.xff_ip target_field: menlo.web.xff_ip ignore_missing: true if: ctx?.json?.event?.xff_ip != "NA" - append: + tag: append_related_ip_8121c591 field: related.ip value: '{{{source.ip}}}' if: ctx.source?.ip != null allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: '{{{destination.ip}}}' if: ctx.destination?.ip != null @@ -351,11 +423,13 @@ processors: } drop(ctx); - append: + tag: append_event_kind_d2f25e35 field: event.kind value: pipeline_error allow_duplicates: false if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/menlo/manifest.yml b/packages/menlo/manifest.yml index 2c9a85606af..4a39d49c817 100644 --- a/packages/menlo/manifest.yml +++ b/packages/menlo/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.3.2" name: menlo title: "Menlo Security" -version: "1.8.1" +version: "1.8.2" source: license: "Elastic-2.0" description: "Collect logs from Menlo Security products with Elastic Agent" diff --git a/packages/microsoft_defender_cloud/changelog.yml b/packages/microsoft_defender_cloud/changelog.yml index 181a0ea0d6d..eb7eac6559b 100644 --- a/packages/microsoft_defender_cloud/changelog.yml +++ b/packages/microsoft_defender_cloud/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.5.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "3.5.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/microsoft_defender_cloud/data_stream/assessment/elasticsearch/ingest_pipeline/default.yml b/packages/microsoft_defender_cloud/data_stream/assessment/elasticsearch/ingest_pipeline/default.yml index 4f3cc247717..096bbcc4828 100644 --- a/packages/microsoft_defender_cloud/data_stream/assessment/elasticsearch/ingest_pipeline/default.yml +++ b/packages/microsoft_defender_cloud/data_stream/assessment/elasticsearch/ingest_pipeline/default.yml @@ -38,7 +38,7 @@ processors: field: event.original tag: json_event_original target_field: json - + # rename to snake case - rename: field: json.name @@ -61,6 +61,7 @@ processors: target_field: json.properties.additionalData.CVEs on_failure: - remove: + tag: remove_json_properties_additionalData_CVEs_ac8868a7 field: json.properties.additionalData.CVEs ignore_missing: true - script: @@ -153,6 +154,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2c910ec6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -162,9 +164,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_cves_base_score_f41df081 field: microsoft_defender_cloud.assessment.additional_data.cves.base_score ignore_missing: true - append: + tag: append_error_message_77372be0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -174,9 +178,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_cves_count_5da348fc field: microsoft_defender_cloud.assessment.additional_data.cves_count ignore_missing: true - append: + tag: append_error_message_8bb7a749 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -186,9 +192,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_max_cvss30_score_204088b8 field: microsoft_defender_cloud.assessment.additional_data.max_cvss30_score ignore_missing: true - append: + tag: append_error_message_30a4d4cc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -199,6 +207,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_052eef4f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -230,9 +239,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_cve_cvss_score_f8f89075 field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.cve.cvss_score ignore_missing: true - append: + tag: append_error_message_ae28eaa4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -242,14 +253,16 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_cve_cvss_version_d26f79e9 field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.cve.cvss_version ignore_missing: true - append: + tag: append_error_message_5e8aab36 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.cve_list - tag: foreach_additional_data_sub_assessment_additional_data_cve_list + tag: foreach_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_cve_list_c68ceb39 if: ctx.microsoft_defender_cloud?.assessment?.additional_data?.sub_assessment?.additional_data?.cve_list instanceof List processor: convert: @@ -264,7 +277,7 @@ processors: ignore_missing: true - foreach: field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.cve_list - tag: foreach_additional_data_sub_assessment_additional_data_cve_list + tag: foreach_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_cve_list_a3fbd685 if: ctx.microsoft_defender_cloud?.assessment?.additional_data?.sub_assessment?.additional_data?.cve_list instanceof List processor: convert: @@ -285,14 +298,16 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_cve_has_public_exploit_e8306d25 field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.cve.has_public_exploit ignore_missing: true - append: + tag: append_error_message_03c235e6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.cve_list - tag: foreach_additional_data_sub_assessment_additional_data_cve_list + tag: foreach_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_cve_list_6a6fe894 if: ctx.microsoft_defender_cloud?.assessment?.additional_data?.sub_assessment?.additional_data?.cve_list instanceof List processor: convert: @@ -313,14 +328,16 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_cve_is_exploit_in_kit_83324a0b field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.cve.is_exploit_in_kit ignore_missing: true - append: + tag: append_error_message_a480e135 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.cve_list - tag: foreach_additional_data_sub_assessment_additional_data_cve_list + tag: foreach_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_cve_list_5d28c0e4 if: ctx.microsoft_defender_cloud?.assessment?.additional_data?.sub_assessment?.additional_data?.cve_list instanceof List processor: convert: @@ -341,14 +358,16 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_cve_is_exploit_verified_8de16b2b field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.cve.is_exploit_verified ignore_missing: true - append: + tag: append_error_message_45390a45 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.cve_list - tag: foreach_additional_data_sub_assessment_additional_data_cve_list + tag: foreach_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_cve_list_a2e24665 if: ctx.microsoft_defender_cloud?.assessment?.additional_data?.sub_assessment?.additional_data?.cve_list instanceof List processor: convert: @@ -369,14 +388,16 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_cve_is_zero_day_beebce03 field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.cve.is_zero_day ignore_missing: true - append: + tag: append_error_message_d67f4fa0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.cve_list - tag: foreach_additional_data_sub_assessment_additional_data_cve_list + tag: foreach_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_cve_list_b077acd4 if: ctx.microsoft_defender_cloud?.assessment?.additional_data?.sub_assessment?.additional_data?.cve_list instanceof List processor: date: @@ -398,14 +419,16 @@ processors: if: ctx.microsoft_defender_cloud?.assessment?.additional_data?.sub_assessment?.additional_data?.cve?.last_modified_date != null && ctx.microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.cve.last_modified_date != '' on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_cve_last_modified_date_7ecedb6f field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.cve.last_modified_date ignore_missing: true - append: + tag: append_error_message_0d5c704b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.cve_list - tag: foreach_additional_data_sub_assessment_additional_data_cve_list + tag: foreach_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_cve_list_c95d3354 if: ctx.microsoft_defender_cloud?.assessment?.additional_data?.sub_assessment?.additional_data?.cve_list instanceof List processor: date: @@ -427,9 +450,11 @@ processors: if: ctx.microsoft_defender_cloud?.assessment?.additional_data?.sub_assessment?.additional_data?.cve?.published_date != null && ctx.microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.cve.published_date != '' on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_cve_published_date_423665df field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.cve.published_date ignore_missing: true - append: + tag: append_error_message_f34c91bb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -439,9 +464,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_cvss_v30_score_8e880dd8 field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.cvss_v30_score ignore_missing: true - append: + tag: append_error_message_97b4e3d5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - json: @@ -465,9 +492,11 @@ processors: if: ctx.microsoft_defender_cloud?.assessment?.additional_data?.sub_assessment?.additional_data?.data?.signature_update_date != null && ctx.microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.data.signature_update_date != '' on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_data_signature_update_date_db62b81f field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.data.signature_update_date ignore_missing: true - append: + tag: append_error_message_130020cb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -479,9 +508,11 @@ processors: if: ctx.microsoft_defender_cloud?.assessment?.additional_data?.sub_assessment?.additional_data?.software_details?.fix_reference?.release_date != null && ctx.microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.software_details.fix_reference.release_date != '' on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_software_details_fix_reference_release_date_2de72367 field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.software_details.fix_reference.release_date ignore_missing: true - append: + tag: append_error_message_846550cb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -491,9 +522,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_software_details_patchable_6e1f5ed7 field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.software_details.patchable ignore_missing: true - append: + tag: append_error_message_87a3bf96 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -514,6 +547,7 @@ processors: } on_failure: - append: + tag: append_error_message_782a17d5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -523,9 +557,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_vulnerability_details_cvss_v2_base_4b33cfd0 field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.vulnerability_details.cvss_v2.base ignore_missing: true - append: + tag: append_error_message_7b8bde81 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -535,9 +571,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_vulnerability_details_cvss_v3_base_011c57d8 field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.vulnerability_details.cvss_v3.base ignore_missing: true - append: + tag: append_error_message_2a7bfe8c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -547,9 +585,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_vulnerability_details_cvss_v4_base_6c5ea180 field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.vulnerability_details.cvss_v4.base ignore_missing: true - append: + tag: append_error_message_240c06bf field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -559,9 +599,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_vulnerability_details_exploitability_assessment_is_in_exploit_kit_177de8ec field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.vulnerability_details.exploitability_assessment.is_in_exploit_kit ignore_missing: true - append: + tag: append_error_message_9dcda12f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -571,9 +613,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_vulnerability_details_exploitability_assessment_is_publicly_disclosed_590e8828 field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.vulnerability_details.exploitability_assessment.is_publicly_disclosed ignore_missing: true - append: + tag: append_error_message_9162158c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -583,9 +627,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_vulnerability_details_exploitability_assessment_is_verified_6c182cd4 field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.vulnerability_details.exploitability_assessment.is_verified ignore_missing: true - append: + tag: append_error_message_4b311d7b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -597,9 +643,11 @@ processors: if: ctx.microsoft_defender_cloud?.assessment?.additional_data?.sub_assessment?.additional_data?.vulnerability_details?.last_modified_date != null && ctx.microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.vulnerability_details.last_modified_date != '' on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_vulnerability_details_last_modified_date_485ac43c field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.vulnerability_details.last_modified_date ignore_missing: true - append: + tag: append_error_message_61242e0f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -611,9 +659,11 @@ processors: if: ctx.microsoft_defender_cloud?.assessment?.additional_data?.sub_assessment?.additional_data?.vulnerability_details?.published_date != null && ctx.microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.vulnerability_details.published_date != '' on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_vulnerability_details_published_date_58758534 field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.vulnerability_details.published_date ignore_missing: true - append: + tag: append_error_message_d6e017c7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -625,9 +675,11 @@ processors: if: ctx.microsoft_defender_cloud?.assessment?.additional_data?.sub_assessment?.time_generated != null && ctx.microsoft_defender_cloud.assessment.additional_data.sub_assessment.time_generated != '' on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_additional_data_sub_assessment_time_generated_ed0c9e03 field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.time_generated ignore_missing: true - append: + tag: append_error_message_58252d85 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -639,9 +691,11 @@ processors: if: ctx.microsoft_defender_cloud?.assessment?.status?.first_evaluation_date != null && ctx.microsoft_defender_cloud.assessment.status.first_evaluation_date != '' on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_status_first_evaluation_date_40c5315d field: microsoft_defender_cloud.assessment.status.first_evaluation_date ignore_missing: true - append: + tag: append_error_message_62641a0d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -653,9 +707,11 @@ processors: if: ctx.microsoft_defender_cloud?.assessment?.status?.status_change_date != null && ctx.microsoft_defender_cloud.assessment.status.status_change_date != '' on_failure: - remove: + tag: remove_microsoft_defender_cloud_assessment_status_status_change_date_bfe65999 field: microsoft_defender_cloud.assessment.status.status_change_date ignore_missing: true - append: + tag: append_error_message_6e3d8439 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -676,7 +732,6 @@ processors: ctx.microsoft_defender_cloud?.assessment?.additional_data?.cves_count == null # populate ECS fields - - set: field: message tag: set_message_from_additional_data_sub_assessment_display_name @@ -688,7 +743,7 @@ processors: copy_from: microsoft_defender_cloud.assessment.display_name ignore_empty_value: true if: ctx.message == null - + # event.* - set: field: event.kind @@ -748,6 +803,7 @@ processors: } on_failure: - append: + tag: append_error_message_683ca916 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -807,6 +863,7 @@ processors: } on_failure: - append: + tag: append_error_message_b4b2b235 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' @@ -826,11 +883,12 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_17119ca5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - grok: field: microsoft_defender_cloud.assessment.resource_details.id - tag: grok_to_extract_cloud_account_id + tag: grok_microsoft_defender_cloud_assessment_resource_details_id_3e4012db patterns: - '^/subscriptions/%{DATA:cloud.account.id}/%{GREEDYDATA}$' - '^%{GREEDYDATA}$' @@ -838,6 +896,7 @@ processors: if: ctx.cloud?.account?.id == null on_failure: - append: + tag: append_error_message_90dc53ba field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -872,7 +931,7 @@ processors: tag: set_cloud_provider_to_azure value: azure if: ctx.microsoft_defender_cloud?.assessment?.resource_details?.source == 'Azure' - + # container.* - set: field: container.name @@ -1005,9 +1064,10 @@ processors: } on_failure: - append: + tag: append_error_message_c509fc4e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - + # orchestrator.* - set: field: orchestrator.resource.parent.type @@ -1084,7 +1144,6 @@ processors: ctx.microsoft_defender_cloud?.assessment?.additional_data?.sub_assessment?.additional_data?.kubernetes_details != null || ctx.microsoft_defender_cloud?.assessment?.additional_data?.sub_assessment?.additional_data?.kubernetes_context != null || ctx.microsoft_defender_cloud?.assessment?.additional_data?.k8s_cluster_id != null - # package.* - set: field: package.name @@ -1180,7 +1239,7 @@ processors: copy_from: microsoft_defender_cloud.assessment.resource_details.resource_type ignore_empty_value: true if: ctx.resource?.sub_type == null - + # result.* - set: field: result.evaluation @@ -1197,7 +1256,7 @@ processors: tag: set_result_evaluation_to_unknown value: unknown if: ctx.event?.outcome == 'unknown' && ctx.microsoft_defender_cloud?.assessment?.class == 'misconfiguration' - + # rule.* - set: field: rule.name @@ -1242,7 +1301,7 @@ processors: copy_from: microsoft_defender_cloud.assessment.additional_data.sub_assessment.category ignore_empty_value: true if: ctx.microsoft_defender_cloud?.assessment?.class == 'misconfiguration' - + # vulnerability.* - set: field: vulnerability.score.base @@ -1312,6 +1371,7 @@ processors: ctx.vulnerability.score.version = version; on_failure: - append: + tag: append_error_message_3eadc0fb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -1336,6 +1396,7 @@ processors: ctx.vulnerability.severity = severity; on_failure: - append: + tag: append_error_message_0b713527 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1378,6 +1439,7 @@ processors: ignore_empty_value: true if: ctx.microsoft_defender_cloud?.assessment?.class == 'vulnerability' - foreach: + tag: foreach_microsoft_defender_cloud_assessment_additional_data_cves_cfb9eabd field: microsoft_defender_cloud.assessment.additional_data.cves if: ctx.vulnerability?.id == null && ctx.microsoft_defender_cloud?.assessment?.additional_data?.cves instanceof List processor: @@ -1398,6 +1460,7 @@ processors: ignore_empty_value: true if: ctx.vulnerability?.published_date == null - foreach: + tag: foreach_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_vulnerability_details_references_51ca223c field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.vulnerability_details.references if: ctx.microsoft_defender_cloud?.assessment?.additional_data?.sub_assessment?.additional_data?.vulnerability_details?.references instanceof List processor: @@ -1424,6 +1487,7 @@ processors: ignore_empty_value: true if: ctx.vulnerability?.cve == null && ctx.microsoft_defender_cloud?.assessment?.additional_data?.sub_assessment?.additional_data?.cve?.title?.startsWith('CVE') == true - foreach: + tag: foreach_microsoft_defender_cloud_assessment_additional_data_sub_assessment_additional_data_cve_list_d1138185 field: microsoft_defender_cloud.assessment.additional_data.sub_assessment.additional_data.cve_list if: ctx.vulnerability?.cve == null && ctx.microsoft_defender_cloud?.assessment?.additional_data?.sub_assessment?.additional_data?.cve_list instanceof List processor: @@ -1433,6 +1497,7 @@ processors: value: '{{{_ingest._value.title}}}' allow_duplicates: false - foreach: + tag: foreach_microsoft_defender_cloud_assessment_additional_data_cves_5c7d1056 field: microsoft_defender_cloud.assessment.additional_data.cves if: ctx.vulnerability?.cve == null && ctx.microsoft_defender_cloud?.assessment?.additional_data?.cves instanceof List processor: @@ -1471,7 +1536,7 @@ processors: tag: set_vulnerability_title_from_package_name value: 'Vulnerability found in {{{package.name}}}' if: ctx.package?.name != null && ctx.vulnerability?.title == null && ctx.microsoft_defender_cloud?.assessment?.class == 'vulnerability' - + # user.* - set: field: user.name @@ -1485,7 +1550,7 @@ processors: copy_from: microsoft_defender_cloud.assessment.additional_data.resource_name ignore_empty_value: true if: ctx.microsoft_defender_cloud?.assessment?.additional_data?.resource_type == 'User' - + # related.* - dissect: description: Extract Digest Hash. @@ -1496,6 +1561,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_48791bf1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -1513,6 +1579,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_de24bc8a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -1533,7 +1600,7 @@ processors: value: '{{{host.name}}}' allow_duplicates: false if: ctx.host?.name != null - + - remove: field: - json @@ -1605,6 +1672,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/microsoft_defender_cloud/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/microsoft_defender_cloud/data_stream/event/elasticsearch/ingest_pipeline/default.yml index 382f316a64d..ad684b60dfd 100644 --- a/packages/microsoft_defender_cloud/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/microsoft_defender_cloud/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -41,6 +41,7 @@ processors: tag: json_to_split_message on_failure: - append: + tag: append_error_message_9eb640d9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -79,7 +80,7 @@ processors: - set: field: event.type value: [indicator] - tag: set_event_type + tag: set_event_type_ff44567c if: | ctx.json?.alerttype != null && [ 'arm_anomalousserviceoperation.credentialaccess', @@ -114,7 +115,7 @@ processors: - set: field: event.category value: [api] - tag: set_event_category + tag: set_event_category_6b727ed8 if: | ctx.json?.alerttype != null && [ 'api_populationspikeinapitraffic', @@ -133,7 +134,7 @@ processors: - set: field: event.category value: [authentication] - tag: set_event_category + tag: set_event_category_df2ba165 if: | ctx.json?.alerttype != null && [ 'vm_loginbruteforcesuccess', @@ -205,7 +206,7 @@ processors: - set: field: event.category value: [configuration] - tag: set_event_category + tag: set_event_category_8fdf0a9d if: | ctx.json?.alerttype != null && [ 'k8s_exposedpostgrestrustauth', @@ -215,7 +216,7 @@ processors: - set: field: event.category value: [malware] - tag: set_event_category + tag: set_event_category_8a7abe38 if: | ctx.json?.alerttype != null && [ 'vm_ammalwarecampaignrelatedexclusion', @@ -291,7 +292,7 @@ processors: - set: field: event.category value: [network] - tag: set_event_category + tag: set_event_category_61003cbe if: | ctx.json?.alerttype != null && [ 'vm_filelessattackbehavior.windows', @@ -331,7 +332,7 @@ processors: - set: field: event.category value: [threat] - tag: set_event_category + tag: set_event_category_66127364 if: | ctx.json?.alerttype != null && [ 'arm_anomalousserviceoperation.credentialaccess', @@ -411,6 +412,7 @@ processors: if: ctx.json?.assessmenteventdataenrichment?.issnapshot != '' on_failure: - append: + tag: append_error_message_a2c22181 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -457,6 +459,7 @@ processors: if: ctx.json?.endtimeutc != null && ctx.json.endtimeutc != '' on_failure: - append: + tag: append_error_message_cdc876e5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -576,6 +579,7 @@ processors: tag: rename_entities_obj ignore_missing: true - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_c4b5cd86 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -586,6 +590,7 @@ processors: tag: append_location_cloud_provider_into_cloud_provider allow_duplicates: false - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_9b2e0b0b field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -601,6 +606,7 @@ processors: field: _ingest._value.creation_time_utc ignore_missing: true - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_4fd88d95 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -616,6 +622,7 @@ processors: field: _ingest._value.end_time_utc ignore_missing: true - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_a5aad92d field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -631,6 +638,7 @@ processors: field: _ingest._value.start_time_utc ignore_missing: true - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_90402973 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -641,6 +649,7 @@ processors: tag: append_container_id_into_container_id allow_duplicates: false - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_2cbc34bb field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -651,6 +660,7 @@ processors: tag: append_domain_name_into_host_domain allow_duplicates: false - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_b1195879 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -661,6 +671,7 @@ processors: tag: append_domain_name_into_related_hosts allow_duplicates: false - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_d4bdf701 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -671,6 +682,7 @@ processors: tag: append_location_city_into_host_geo_city_name allow_duplicates: false - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_6a3ae623 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -681,6 +693,7 @@ processors: tag: append_location_country_code_into_host_geo_country_iso_code allow_duplicates: false - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_c030255b field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -691,6 +704,7 @@ processors: tag: append_location_country_name_into_host_geo_country_name allow_duplicates: false - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_00203e48 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -708,6 +722,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_d200e6ab field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -725,6 +740,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_d82735fb field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -735,6 +751,7 @@ processors: tag: append_host_name_into_host_hostname allow_duplicates: false - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_4fc982c1 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -745,6 +762,7 @@ processors: tag: append_host_name_into_related_hosts allow_duplicates: false - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_0b278598 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -755,6 +773,7 @@ processors: tag: append_os_family_into_host_os_family allow_duplicates: false - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_a01a9551 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -764,6 +783,7 @@ processors: tag: lowercase_protocol ignore_missing: true - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_964de64b field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -774,6 +794,7 @@ processors: tag: append_protocol_into_network_transport allow_duplicates: false - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_718a7ffd field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -784,6 +805,7 @@ processors: tag: append_process_id_into_process_entity_id allow_duplicates: false - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_6a03db08 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -801,6 +823,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_584b51f9 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -811,6 +834,7 @@ processors: tag: append_address_into_related_ip allow_duplicates: false - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_dacdf1e0 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -833,6 +857,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_63a71602 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -848,6 +873,7 @@ processors: tag: append_address_into_related_ip allow_duplicates: false - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_991075d0 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -865,6 +891,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_a25682f4 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -887,6 +914,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_dec0d89c field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -909,6 +937,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_1ead40bf field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -926,6 +955,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_82d43569 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -943,6 +973,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_480517c4 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -960,6 +991,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_177074c5 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -969,6 +1001,7 @@ processors: tag: uri_parts_url ignore_failure: true - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_9cf140e6 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List ignore_failure: true @@ -1079,6 +1112,7 @@ processors: if: ctx.json?.isincident != '' on_failure: - append: + tag: append_error_message_c8eab27f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1105,6 +1139,7 @@ processors: if: ctx.json?.processingendtime != null && ctx.json.processingendtime != '' on_failure: - append: + tag: append_error_message_9ba22166 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1156,6 +1191,7 @@ processors: if: ctx.json?.properties?.definition?.properties?.assessmentdefinitions != '' on_failure: - append: + tag: append_error_message_18765014 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1172,6 +1208,7 @@ processors: if: ctx.json?.properties?.definition?.properties?.maxscore != '' on_failure: - append: + tag: append_error_message_30a188fa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1208,6 +1245,7 @@ processors: if: ctx.json?.properties?.failedresources != '' on_failure: - append: + tag: append_error_message_6f83ba39 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1219,6 +1257,7 @@ processors: if: ctx.json?.properties?.healthyresourcecount != '' on_failure: - append: + tag: append_error_message_b3520747 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1275,6 +1314,7 @@ processors: if: ctx.json?.properties?.metadata?.preview != '' on_failure: - append: + tag: append_error_message_dafde513 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1306,6 +1346,7 @@ processors: if: ctx.json?.properties?.notapplicableresourcecount != '' on_failure: - append: + tag: append_error_message_39f8add3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1317,6 +1358,7 @@ processors: if: ctx.json?.properties?.passedresources != '' on_failure: - append: + tag: append_error_message_f807d347 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1368,6 +1410,7 @@ processors: if: ctx.json?.properties?.score?.current != '' on_failure: - append: + tag: append_error_message_4c2dba6f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1379,6 +1422,7 @@ processors: if: ctx.json?.properties?.score?.max != '' on_failure: - append: + tag: append_error_message_17c6650f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1390,6 +1434,7 @@ processors: if: ctx.json?.properties?.score?.percentage != '' on_failure: - append: + tag: append_error_message_d41dc3af field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1401,6 +1446,7 @@ processors: if: ctx.json?.properties?.skippedresources != '' on_failure: - append: + tag: append_error_message_4f763df7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1442,6 +1488,7 @@ processors: if: ctx.json?.properties?.status?.firstevaluationdate != null && ctx.json.properties.status.firstevaluationdate != '' on_failure: - append: + tag: append_error_message_ef877010 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1458,6 +1505,7 @@ processors: if: ctx.json?.properties?.status?.statuschangedate != null && ctx.json.properties.status.statuschangedate != '' on_failure: - append: + tag: append_error_message_74cd56ce field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -1469,6 +1517,7 @@ processors: if: ctx.json?.properties?.timegenerated != null && ctx.json.properties.timegenerated != '' on_failure: - append: + tag: append_error_message_3b190188 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1480,6 +1529,7 @@ processors: if: ctx.json?.properties?.unhealthyresourcecount != '' on_failure: - append: + tag: append_error_message_6224f40b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1491,6 +1541,7 @@ processors: if: ctx.json?.properties?.weight != '' on_failure: - append: + tag: append_error_message_63095a8b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1532,6 +1583,7 @@ processors: if: ctx.json?.securityeventdataenrichment?.issnapshot != '' on_failure: - append: + tag: append_error_message_cf923151 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1558,6 +1610,7 @@ processors: } on_failure: - append: + tag: append_error_message_7d3f1610 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -1569,6 +1622,7 @@ processors: if: ctx.json?.starttimeutc != null && ctx.json.starttimeutc != '' on_failure: - append: + tag: append_error_message_235a806b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1605,6 +1659,7 @@ processors: if: ctx.json?.subassessmenteventdataenrichment?.issnapshot != '' on_failure: - append: + tag: append_error_message_31c969ac field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1631,6 +1686,7 @@ processors: if: ctx.json?.timegenerated != null && ctx.json.timegenerated != '' on_failure: - append: + tag: append_error_message_966b8b0f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -1641,12 +1697,13 @@ processors: if: ctx.json?.timegenerated != null && ctx.json.timegenerated != '' on_failure: - append: + tag: append_error_message_36145f88 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: field: json.type target_field: microsoft_defender_cloud.event.type - tag: rename_type + tag: rename_json_type_to_microsoft_defender_cloud_event_type_1a6e2383 ignore_missing: true - rename: field: json.vendorname @@ -1674,6 +1731,7 @@ processors: tag: rename_workspace_subscription_id ignore_missing: true - foreach: + tag: foreach_microsoft_defender_cloud_event_entities_c0b82890 field: microsoft_defender_cloud.event.entities if: ctx.microsoft_defender_cloud?.event?.entities instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) ignore_failure: true @@ -1730,6 +1788,7 @@ processors: tag: set_pipeline_error_into_event_kind if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/microsoft_defender_cloud/manifest.yml b/packages/microsoft_defender_cloud/manifest.yml index 196870b90b4..ad536485b4b 100644 --- a/packages/microsoft_defender_cloud/manifest.yml +++ b/packages/microsoft_defender_cloud/manifest.yml @@ -1,7 +1,7 @@ format_version: '3.3.2' name: microsoft_defender_cloud title: Microsoft Defender for Cloud -version: '3.5.1' +version: '3.5.2' description: Collect logs from Microsoft Defender for Cloud with Elastic Agent. type: integration categories: diff --git a/packages/microsoft_defender_endpoint/changelog.yml b/packages/microsoft_defender_endpoint/changelog.yml index eeebeecdc43..24e8dda0ae3 100644 --- a/packages/microsoft_defender_endpoint/changelog.yml +++ b/packages/microsoft_defender_endpoint/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "4.9.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "4.9.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/microsoft_defender_endpoint/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/microsoft_defender_endpoint/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 1e382a8b414..fd7419da892 100644 --- a/packages/microsoft_defender_endpoint/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/microsoft_defender_endpoint/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -13,30 +13,37 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_e947c330 field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field. Helps to maintain compatibility with the Logstash `ecs_compatibility feature`.' - json: + tag: json_event_original_to_json_5e54dc16 field: event.original target_field: json - drop: + tag: drop_1101a70f if: ctx.json?.value?.isEmpty() == true - remove: + tag: remove_65a5568f field: - json.comments - host - cloud ignore_missing: true - rename: + tag: rename_json_evidence_processCommandLine_to_process_command_line_87d5c44c field: json.evidence.processCommandLine target_field: process.command_line ignore_missing: true @@ -94,13 +101,15 @@ processors: } on_failure: - append: + tag: append_error_message_6ff52099 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' -######################### -## ECS General Mapping ## -######################### + ######################### + ## ECS General Mapping ## + ######################### - script: + tag: script_689b9659 lang: painless if: ctx.json != null params: @@ -114,6 +123,7 @@ processors: ctx.json.entrySet().removeIf(entry -> params.values.contains(entry.getValue())); } - script: + tag: script_175274f4 lang: painless if: ctx.json?.evidence != null params: @@ -140,79 +150,98 @@ processors: } drop(ctx); - set: + tag: set_cloud_provider_136a398d field: cloud.provider value: azure - set: + tag: set_timestamp_eb1f6f42 field: '@timestamp' value: '{{{json.alertUpdateTime}}}' if: ctx.json?.alertUpdateTime != null - rename: + tag: rename_json_aadTenantId_to_cloud_account_id_8f6f7734 field: json.aadTenantId target_field: cloud.account.id ignore_missing: true - rename: + tag: rename_json_machineId_to_cloud_instance_id_340e5847 field: json.machineId target_field: cloud.instance.id ignore_missing: true - rename: + tag: rename_json_title_to_message_81e913ab field: json.title target_field: message ignore_missing: true -####################### -## ECS Event Mapping ## -####################### + ####################### + ## ECS Event Mapping ## + ####################### - set: + tag: set_event_kind_39295792 field: event.kind value: alert -# Events returned from the API is always in UTC, so should never use anything else + # Events returned from the API is always in UTC, so should never use anything else - set: + tag: set_event_timezone_6275b231 field: event.timezone value: UTC - set: + tag: set_event_action_c3bce720 field: event.action value: '{{{json.category}}}' if: ctx.json?.category != null - set: + tag: set_event_provider_c7a88896 field: event.provider value: defender_endpoint - set: + tag: set_event_created_749d01ee field: event.created value: '{{{json.alertCreationTime}}}' if: ctx.json?.alertCreationTime != null - append: + tag: append_event_category_3fa422a0 field: event.category value: host - append: + tag: append_event_category_dc037e29 field: event.category value: malware if: ctx.json?.category == 'Malware' - append: + tag: append_event_category_af56ec52 field: event.category value: process if: ctx.json?.evidence?.entityType == 'Process' - append: + tag: append_event_type_fc8e2967 field: event.type value: access if: ctx.json?.evidence?.entityType == 'User' - append: + tag: append_event_type_396d75c1 field: event.type value: - start if: ctx.json?.status == 'New' - append: + tag: append_event_type_b517d614 field: event.type value: end if: ctx.json?.status == 'Resolved' - rename: + tag: rename_json_id_to_event_id_6438e920 field: json.id target_field: event.id ignore_missing: true - rename: + tag: rename_json_firstEventTime_to_event_start_20a87aa3 field: json.firstEventTime target_field: event.start ignore_missing: true - rename: + tag: rename_json_lastEventTime_to_event_end_50b3aea0 field: json.lastEventTime target_field: event.end ignore_missing: true @@ -235,69 +264,81 @@ processors: } on_failure: - append: + tag: append_error_message_3cfc5ed3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: + tag: script_6763c8b3 lang: painless if: ctx.event?.start != null && ctx.event?.end != null source: > Instant eventstart = ZonedDateTime.parse(ctx.event.start).toInstant(); Instant eventend = ZonedDateTime.parse(ctx.event.end).toInstant(); ctx.event['duration'] = ChronoUnit.NANOS.between(eventstart, eventend); - -######################## -## ECS Threat Mapping ## -######################## + ######################## + ## ECS Threat Mapping ## + ######################## - set: + tag: set_threat_framework_3cca1f4f field: threat.framework value: MITRE ATT&CK if: ctx.json?.category != null - append: + tag: append_threat_technique_name_e694defd field: threat.technique.name value: '{{{json.category}}}' if: ctx.json?.category != null - rename: + tag: rename_json_description_to_rule_description_51d58c52 field: json.description target_field: rule.description ignore_missing: true if: ctx.json?.description != null && ctx.json.description.length() < 1020 -###################### -## ECS File Mapping ## -###################### + ###################### + ## ECS File Mapping ## + ###################### - rename: + tag: rename_json_evidence_fileName_to_file_name_37b76119 field: json.evidence.fileName target_field: file.name ignore_missing: true - rename: + tag: rename_json_evidence_sha256_to_file_hash_sha256_d8376191 field: json.evidence.sha256 target_field: file.hash.sha256 ignore_missing: true - rename: + tag: rename_json_evidence_sha1_to_file_hash_sha1_c9341a59 field: json.evidence.sha1 target_field: file.hash.sha1 ignore_missing: true - rename: + tag: rename_json_evidence_filePath_to_file_path_c0b894ad field: json.evidence.filePath target_field: file.path ignore_missing: true -###################### -## ECS Process Mapping ## -###################### + ###################### + ## ECS Process Mapping ## + ###################### - rename: + tag: rename_json_evidence_processId_to_process_pid_63c8c33f field: json.evidence.processId target_field: process.pid ignore_missing: true - rename: + tag: rename_json_evidence_processCreationTime_to_process_start_5744afe1 field: json.evidence.processCreationTime target_field: process.start ignore_missing: true - rename: + tag: rename_json_evidence_parentProcessId_to_process_parent_pid_9fff990d field: json.evidence.parentProcessId target_field: process.parent.pid ignore_missing: true - rename: + tag: rename_json_evidence_parentProcessCreationTime_to_process_parent_start_5e342db5 field: json.evidence.parentProcessCreationTime target_field: process.parent.start ignore_missing: true @@ -311,95 +352,112 @@ processors: tag: set_process_parent_entity_id value: '{{{process.parent.pid}}}' ignore_empty_value: true -########################## -## ECS Observer Mapping ## -########################## + ########################## + ## ECS Observer Mapping ## + ########################## - set: + tag: set_observer_product_d661edd2 field: observer.product value: Defender for Endpoint - set: + tag: set_observer_vendor_99db8544 field: observer.vendor value: Microsoft - rename: + tag: rename_json_detectionSource_to_observer_name_a0edeb13 field: json.detectionSource target_field: observer.name ignore_missing: true -##################### -## ECS URL Mapping ## -##################### + ##################### + ## ECS URL Mapping ## + ##################### - rename: + tag: rename_json_evidence_url_to_url_full_62e6e9fd field: json.evidence.url target_field: url.full ignore_missing: true if: ctx.json?.evidence?.url != null - uri_parts: + tag: uri_parts_url_full_a1687495 field: url.full ignore_failure: true if: ctx.url?.full != null -###################### -## ECS Host Mapping ## -###################### + ###################### + ## ECS Host Mapping ## + ###################### - rename: + tag: rename_json_computerDnsName_to_host_hostname_4eaae100 field: json.computerDnsName target_field: host.hostname ignore_missing: true - lowercase: + tag: lowercase_host_hostname_to_host_name_39c0d2d1 target_field: host.name field: host.hostname if: ctx.host?.hostname != null - set: + tag: set_host_id_c0d880eb field: host.id copy_from: cloud.instance.id if: ctx.cloud?.instance?.id != null ignore_empty_value: true -###################### -## ECS User Mapping ## -###################### + ###################### + ## ECS User Mapping ## + ###################### - rename: + tag: rename_json_relatedUser_userName_to_user_name_8ca12340 field: json.relatedUser.userName target_field: user.name ignore_missing: true - rename: + tag: rename_json_relatedUser_domainName_to_user_domain_53e84dba field: json.relatedUser.domainName target_field: user.domain ignore_missing: true - rename: + tag: rename_json_evidence_userSid_to_user_id_2390c9a2 field: json.evidence.userSid target_field: user.id ignore_missing: true -######################### -## ECS Related Mapping ## -######################### + ######################### + ## ECS Related Mapping ## + ######################### - append: + tag: append_related_ip_8834607b field: related.ip value: '{{{json.evidence.ipAddress}}}' if: ctx.json?.evidence?.ipAddress != null - append: + tag: append_related_user_c036eec2 field: related.user value: '{{{user.name}}}' if: ctx.user?.name != null - append: + tag: append_related_hash_12718026 field: related.hash value: '{{{file.hash.sha1}}}' if: ctx.file?.hash?.sha1 != null - append: + tag: append_related_hash_7574f0ee field: related.hash value: '{{{file.hash.sha256}}}' if: ctx.file?.hash?.sha256 != null - append: + tag: append_related_hosts_555b812b field: related.hosts value: '{{{host.name}}}' if: ctx.host?.name != null && ctx.host.name != '' allow_duplicates: false -############# -## Cleanup ## -############# + ############# + ## Cleanup ## + ############# - remove: + tag: remove_2189f235 field: - json.alertCreationTime - json.severity @@ -407,14 +465,17 @@ processors: - json.category ignore_missing: true - rename: + tag: rename_json_to_microsoft_defender_endpoint_63b0021f field: json target_field: microsoft.defender_endpoint ignore_missing: true - convert: + tag: convert_microsoft_defender_endpoint_incidentId_e9e01906 field: microsoft.defender_endpoint.incidentId type: string ignore_missing: true - convert: + tag: convert_microsoft_defender_endpoint_investigationId_cfb73590 field: microsoft.defender_endpoint.investigationId type: string ignore_missing: true diff --git a/packages/microsoft_defender_endpoint/data_stream/machine/elasticsearch/ingest_pipeline/default.yml b/packages/microsoft_defender_endpoint/data_stream/machine/elasticsearch/ingest_pipeline/default.yml index 46d248a2661..0e941b9826f 100644 --- a/packages/microsoft_defender_endpoint/data_stream/machine/elasticsearch/ingest_pipeline/default.yml +++ b/packages/microsoft_defender_endpoint/data_stream/machine/elasticsearch/ingest_pipeline/default.yml @@ -39,6 +39,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -99,7 +100,7 @@ processors: ignore_empty_value: true - set: field: host.hostname - tag: set_host_hostname_from_machine_computer_dns_name + tag: set_host_hostname_ab5e048e copy_from: microsoft_defender_endpoint.machine.computer_dns_name ignore_empty_value: true - append: @@ -132,6 +133,7 @@ processors: if: ctx.json?.firstSeen != null && ctx.json.firstSeen != '' on_failure: - append: + tag: append_error_message_a1e3ceb3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -175,6 +177,7 @@ processors: } drop(ctx.json.ipAddresses); - foreach: + tag: foreach_json_ipAddresses_87c45d1d field: json.ipAddresses if: ctx.json?.ipAddresses instanceof List processor: @@ -189,6 +192,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_ipAddresses_a6fb5ad7 field: json.ipAddresses if: ctx.json?.ipAddresses instanceof List processor: @@ -198,6 +202,7 @@ processors: value: '{{{_ingest._value.ip_address}}}' allow_duplicates: false - foreach: + tag: foreach_json_ipAddresses_baea1cc0 field: json.ipAddresses if: ctx.json?.ipAddresses instanceof List processor: @@ -207,6 +212,7 @@ processors: target_field: _ingest._value.mac_address ignore_missing: true - foreach: + tag: foreach_json_ipAddresses_c189e9c7 field: json.ipAddresses if: ctx.json?.ipAddresses instanceof List processor: @@ -217,6 +223,7 @@ processors: tag: gsub_ipAddresses_mac_address ignore_missing: true - foreach: + tag: foreach_json_ipAddresses_814a16e5 field: json.ipAddresses if: ctx.json?.ipAddresses instanceof List processor: @@ -226,6 +233,7 @@ processors: target_field: _ingest._value.operational_status ignore_missing: true - foreach: + tag: foreach_json_ipAddresses_bdee6b23 field: json.ipAddresses if: ctx.json?.ipAddresses instanceof List processor: @@ -248,6 +256,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ad2f38d6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -258,6 +267,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_fc65d810 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -268,6 +278,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d4c62f03 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -279,6 +290,7 @@ processors: if: ctx.json?.lastExternalIpAddress != '' on_failure: - append: + tag: append_error_message_85663805 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -307,6 +319,7 @@ processors: if: ctx.json?.lastIpAddress != '' on_failure: - append: + tag: append_error_message_e8e64b8a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -324,6 +337,7 @@ processors: if: ctx.json?.lastSeen != null && ctx.json.lastSeen != '' on_failure: - append: + tag: append_error_message_1449c1b1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -365,6 +379,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5b6ca096 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -425,6 +440,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8f72e2e0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -463,6 +479,7 @@ processors: copy_from: microsoft_defender_endpoint.machine.version ignore_empty_value: true - set: + tag: set_host_os_name_0240352b field: host.os.name value: '{{{host.os.platform}}} {{{host.os.version}}}' ignore_failure: true @@ -539,6 +556,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/microsoft_defender_endpoint/data_stream/machine_action/elasticsearch/ingest_pipeline/default.yml b/packages/microsoft_defender_endpoint/data_stream/machine_action/elasticsearch/ingest_pipeline/default.yml index b85ee043a49..d75aeb8e12c 100644 --- a/packages/microsoft_defender_endpoint/data_stream/machine_action/elasticsearch/ingest_pipeline/default.yml +++ b/packages/microsoft_defender_endpoint/data_stream/machine_action/elasticsearch/ingest_pipeline/default.yml @@ -39,6 +39,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -59,6 +60,7 @@ processors: target_field: microsoft_defender_endpoint.machine_action.type ignore_missing: true - set: + tag: set_event_action_96d6634e field: event.action copy_from: microsoft_defender_endpoint.machine_action.type ignore_failure: true @@ -118,11 +120,13 @@ processors: target_field: microsoft_defender_endpoint.machine_action.status ignore_missing: true - set: + tag: set_event_outcome_3a585902 field: event.outcome value: success if: ctx.microsoft_defender_endpoint?.machine_action?.status != null && ctx.microsoft_defender_endpoint.machine_action.status.toLowerCase() == 'succeeded' ignore_failure: true - set: + tag: set_event_outcome_df272abf field: event.outcome value: failure if: ctx.microsoft_defender_endpoint?.machine_action?.status != null && ctx.microsoft_defender_endpoint.machine_action.status.toLowerCase() == 'failed' @@ -141,6 +145,7 @@ processors: if: ctx.json?.cancellationDateTimeUtc != null && ctx.json.cancellationDateTimeUtc != '' on_failure: - append: + tag: append_error_message_038f4e55 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -171,7 +176,7 @@ processors: ignore_empty_value: true - set: field: host.hostname - tag: set_host_name_from_machine_action_computer_dns_name + tag: set_host_hostname_d388b1b4 copy_from: microsoft_defender_endpoint.machine_action.computer_dns_name ignore_empty_value: true - append: @@ -189,6 +194,7 @@ processors: if: ctx.json?.creationDateTimeUtc != null && ctx.json.creationDateTimeUtc != '' on_failure: - append: + tag: append_error_message_401a27c7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -209,6 +215,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_fc41d848 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -241,6 +248,7 @@ processors: if: ctx.json?.lastUpdateDateTimeUtc != null && ctx.json.lastUpdateDateTimeUtc != '' on_failure: - append: + tag: append_error_message_cf43d1d0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -284,7 +292,7 @@ processors: description: Set file.hash.sha1, file.hash.md5, file.hash.sha256. tag: script_set_file_hash_* lang: painless - if : ctx.microsoft_defender_endpoint?.machine_action?.related_file_info?.file_identifier_type != null + if: ctx.microsoft_defender_endpoint?.machine_action?.related_file_info?.file_identifier_type != null source: > ctx.file = ctx.file ?: [:]; ctx.file.hash = ctx.file.hash ?: [:]; @@ -299,6 +307,7 @@ processors: } on_failure: - append: + tag: append_error_message_389755e8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -410,6 +419,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/microsoft_defender_endpoint/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml b/packages/microsoft_defender_endpoint/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml index 89ecc8bc5a5..4334a700ae3 100644 --- a/packages/microsoft_defender_endpoint/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml +++ b/packages/microsoft_defender_endpoint/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml @@ -42,6 +42,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -83,7 +84,7 @@ processors: value: Microsoft Defender for Endpoint - set: field: observer.vendor - tag : set_observer_vendor + tag: set_observer_vendor value: Microsoft - rename: field: json.CveBatchTitle @@ -141,6 +142,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f24cecb0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -250,6 +252,7 @@ processors: if: ctx.json?.FirstSeenTimestamp != null && ctx.json.FirstSeenTimestamp != '' on_failure: - append: + tag: append_error_message_510fbcef field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -276,6 +279,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_609ac007 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -289,6 +293,7 @@ processors: if: ctx.json?.LastSeenTimestamp != null && ctx.json.LastSeenTimestamp != '' on_failure: - append: + tag: append_error_message_9e00bbc3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -347,6 +352,7 @@ processors: copy_from: microsoft_defender_endpoint.vulnerability.os_version ignore_empty_value: true - set: + tag: set_host_os_name_0240352b field: host.os.name value: '{{{host.os.platform}}} {{{host.os.version}}}' ignore_failure: true @@ -359,6 +365,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8b748de0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -394,6 +401,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4ee989b8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -468,7 +476,7 @@ processors: ignore_empty_value: true - remove: field: - - microsoft_defender_endpoint.vulnerability.cve_batch_title + - microsoft_defender_endpoint.vulnerability.cve_batch_title - microsoft_defender_endpoint.vulnerability.device_id - microsoft_defender_endpoint.vulnerability.device_name - microsoft_defender_endpoint.vulnerability.id @@ -520,10 +528,11 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false - if: ctx.error?.message != null + if: ctx.error?.message != null on_failure: - append: field: error.message diff --git a/packages/microsoft_defender_endpoint/manifest.yml b/packages/microsoft_defender_endpoint/manifest.yml index 4c1cc3434ca..ae46f331120 100644 --- a/packages/microsoft_defender_endpoint/manifest.yml +++ b/packages/microsoft_defender_endpoint/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.4.0" name: microsoft_defender_endpoint title: Microsoft Defender for Endpoint -version: "4.9.1" +version: "4.9.2" description: Collect logs from Microsoft Defender for Endpoint with Elastic Agent. categories: - security diff --git a/packages/microsoft_exchange_online_message_trace/changelog.yml b/packages/microsoft_exchange_online_message_trace/changelog.yml index 1295332affd..8955e7fe350 100644 --- a/packages/microsoft_exchange_online_message_trace/changelog.yml +++ b/packages/microsoft_exchange_online_message_trace/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.2.6" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "2.2.5" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/microsoft_exchange_online_message_trace/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/microsoft_exchange_online_message_trace/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 153a27ffabd..00fa191b07e 100644 --- a/packages/microsoft_exchange_online_message_trace/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/microsoft_exchange_online_message_trace/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -2,6 +2,7 @@ description: Pipeline for processing Message Trace Logs processors: - rename: + tag: rename_message_to_event_original_86cdcc50 field: message target_field: event.original ignore_missing: true @@ -14,63 +15,79 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' # Parsing of payload json - json: + tag: json_event_original_to_microsoft_online_message_trace_f8c7efed field: event.original target_field: microsoft.online_message_trace # Map field names from the Graph API to equivalents from the previous API - rename: + tag: rename_microsoft_online_message_trace_fromIP_to_microsoft_online_message_trace_FromIP_58bb58aa field: microsoft.online_message_trace.fromIP target_field: microsoft.online_message_trace.FromIP ignore_missing: true - rename: + tag: rename_microsoft_online_message_trace_messageId_to_microsoft_online_message_trace_MessageId_2efcdcb4 field: microsoft.online_message_trace.messageId target_field: microsoft.online_message_trace.MessageId ignore_missing: true - rename: + tag: rename_microsoft_online_message_trace_id_to_microsoft_online_message_trace_MessageTraceId_5828108a field: microsoft.online_message_trace.id target_field: microsoft.online_message_trace.MessageTraceId ignore_missing: true - rename: + tag: rename_microsoft_online_message_trace_receivedDateTime_to_microsoft_online_message_trace_Received_7de5a4d3 field: microsoft.online_message_trace.receivedDateTime target_field: microsoft.online_message_trace.Received ignore_missing: true - rename: + tag: rename_microsoft_online_message_trace_recipientAddress_to_microsoft_online_message_trace_RecipientAddress_c6aaf3be field: microsoft.online_message_trace.recipientAddress target_field: microsoft.online_message_trace.RecipientAddress ignore_missing: true - rename: + tag: rename_microsoft_online_message_trace_senderAddress_to_microsoft_online_message_trace_SenderAddress_1ea16ac8 field: microsoft.online_message_trace.senderAddress target_field: microsoft.online_message_trace.SenderAddress ignore_missing: true - rename: + tag: rename_microsoft_online_message_trace_size_to_microsoft_online_message_trace_Size_2931f41e field: microsoft.online_message_trace.size target_field: microsoft.online_message_trace.Size ignore_missing: true - rename: + tag: rename_microsoft_online_message_trace_status_to_microsoft_online_message_trace_Status_e7bd64bc field: microsoft.online_message_trace.status target_field: microsoft.online_message_trace.Status ignore_missing: true - rename: + tag: rename_microsoft_online_message_trace_subject_to_microsoft_online_message_trace_Subject_3250b694 field: microsoft.online_message_trace.subject target_field: microsoft.online_message_trace.Subject ignore_missing: true - rename: + tag: rename_microsoft_online_message_trace_toIP_to_microsoft_online_message_trace_ToIP_d866f2e4 field: microsoft.online_message_trace.toIP target_field: microsoft.online_message_trace.ToIP ignore_missing: true # Drop "null" event - drop: - if: ctx.microsoft?.online_message_trace?.value instanceof List && ctx.microsoft.online_message_trace.value.length == 0 + tag: drop_e6f11cc5 + if: ctx.microsoft?.online_message_trace?.value instanceof List && ctx.microsoft.online_message_trace.value.length == 0 - drop: + tag: drop_ec724f95 if: ctx.microsoft?.online_message_trace?.Status?.equalsIgnoreCase(ctx._conf?.drop_status) == true - set: + tag: set_event_outcome_2023d459 field: event.outcome value: success if: ctx.microsoft?.online_message_trace?.Status?.equalsIgnoreCase('delivered') == true - set: + tag: set_event_outcome_e25919b7 field: event.outcome value: failure if: > @@ -78,94 +95,116 @@ processors: ctx.microsoft?.online_message_trace?.Status?.equalsIgnoreCase('quarantined') == true || ctx.microsoft?.online_message_trace?.Status?.equalsIgnoreCase('filteredAsSpam') == true - set: + tag: set_event_outcome_6f947b7c field: event.outcome value: unknown if: ctx.event?.outcome == null # Event Category & Type https://www.elastic.co/guide/en/ecs/current/ecs-allowed-values-event-category.html#ecs-event-category-email - set: + tag: set_event_category_4861796c field: event.category value: [email] - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] - set: + tag: set__temp__email_from_address_b7bf0e97 if: ctx.microsoft?.online_message_trace?.SenderAddress != "<>" field: _temp_.email.from.address copy_from: microsoft.online_message_trace.SenderAddress ignore_empty_value: true - append: + tag: append_email_from_address_50a9aee0 field: email.from.address value: '{{{_temp_.email.from.address}}}' if: ctx._temp_?.email?.from?.address != null - set: + tag: set_source_user_id_897d343c field: source.user.id copy_from: _temp_.email.from.address ignore_empty_value: true - set: + tag: set_source_user_email_292d32d5 field: source.user.email copy_from: _temp_.email.from.address ignore_empty_value: true - set: + tag: set_email_local_id_1003b996 field: email.local_id copy_from: microsoft.online_message_trace.MessageTraceId ignore_empty_value: true - set: + tag: set_destination_ip_b7b3c99b field: destination.ip copy_from: microsoft.online_message_trace.ToIP ignore_empty_value: true - set: + tag: set_email_attachments_file_size_629523d0 field: email.attachments.file.size copy_from: microsoft.online_message_trace.Size ignore_empty_value: true - set: + tag: set_email_delivery_timestamp_463b387a field: email.delivery_timestamp copy_from: microsoft.online_message_trace.Received ignore_empty_value: true - set: + tag: set_email_subject_c18b7f84 field: email.subject copy_from: microsoft.online_message_trace.Subject ignore_empty_value: true - set: + tag: set__temp__email_to_address_b8b42654 field: _temp_.email.to.address copy_from: microsoft.online_message_trace.RecipientAddress ignore_empty_value: true - append: + tag: append_email_to_address_ef9a0133 field: email.to.address value: '{{{_temp_.email.to.address}}}' if: ctx._temp_?.email?.to?.address != null - set: + tag: set_destination_user_id_ee80b5cd field: destination.user.id copy_from: microsoft.online_message_trace.RecipientAddress ignore_empty_value: true - set: + tag: set_destination_user_email_3baf170e field: destination.user.email copy_from: microsoft.online_message_trace.RecipientAddress ignore_empty_value: true - set: + tag: set_source_ip_57b71e55 field: source.ip copy_from: microsoft.online_message_trace.FromIP ignore_empty_value: true - set: + tag: set_email_message_id_0d658bad field: email.message_id copy_from: microsoft.online_message_trace.MessageId ignore_empty_value: true # Parse the date included in logs - date: + tag: date_email_delivery_timestamp_8b3059d1 field: "email.delivery_timestamp" formats: - "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSZ" - "ISO8601" # IP Geolocation Lookup - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true # IP Autonomous System (AS) Lookup - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -174,14 +213,17 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -190,10 +232,12 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true @@ -213,13 +257,16 @@ processors: ignore_failure: false ignore_missing: true - registered_domain: + tag: registered_domain_destination_domain_to_destination_1322951c field: "destination.domain" target_field: "destination" - registered_domain: + tag: registered_domain_source_domain_to_source_2342224a field: "source.domain" target_field: "source" # Calculation of mail Direction - script: + tag: script_34585be1 lang: painless description: Script to extract the direction of a mailflow if: ctx._conf?.local_domains != null @@ -242,64 +289,77 @@ processors: else {ctx.email.direction = "unknown";} # Extract Username from mail - append: + tag: append_related_user_c8274169 field: related.user value: '{{{_temp_.email.to.address}}}' allow_duplicates: false if: ctx._temp_?.email?.to?.address != null && ctx._temp_.email.to.address != "" - append: + tag: append_related_user_5d58d760 field: related.user value: '{{{_temp_.email.from.address}}}' allow_duplicates: false if: ctx._temp_?.email?.from?.address != null && ctx._temp_.email.from.address != "" - dissect: + tag: dissect__temp__email_to_address_2d013dc1 field: _temp_.email.to.address pattern: '%{_temp_.to_user_name}@%{_temp_.to_user_domain}' ignore_missing: true - append: + tag: append_related_user_ce5e784f field: related.user value: '{{{_temp_.to_user_name}}}' allow_duplicates: false if: ctx._temp_?.to_user_name != null && ctx._temp_.to_user_name != "" - dissect: + tag: dissect__temp__email_from_address_24dc2524 field: _temp_.email.from.address pattern: '%{_temp_.from_user_name}@%{_temp_.from_user_domain}' ignore_missing: true - append: + tag: append_related_user_e08ebfbc field: related.user value: '{{{_temp_.from_user_name}}}' allow_duplicates: false if: ctx._temp_?.from_user_name != null && ctx._temp_.from_user_name != "" - append: + tag: append_user_name_1a617eaf field: user.name value: '{{{_temp_.to_user_name}}}' allow_duplicates: false if: ctx._temp_?.to_user_name != null && ctx._temp_.to_user_name != "" && ctx.email?.direction == "inbound" || ctx.email?.direction == "internal" - append: + tag: append_user_name_17a1bd37 field: user.name value: '{{{_temp_.from_user_name}}}' allow_duplicates: false if: ctx._temp_?.from_user_name != null && ctx._temp_.from_user_name != "" && ctx.email?.direction == "outbound" || ctx.email?.direction == "internal" - append: + tag: append_user_email_e10ffe9e field: user.email value: '{{{_temp_.email.to.address}}}' allow_duplicates: false if: ctx._temp_?.email?.to?.address != null && ctx._temp_.email.to.address != "" && ctx.email?.direction == "inbound" || ctx.email?.direction == "internal" - append: + tag: append_user_email_5b7a9fe2 field: user.email value: '{{{_temp_.email.from.address}}}' allow_duplicates: false if: ctx._temp_?.email?.from?.address != null && ctx._temp_.email.from.address != "" && ctx.email?.direction == "outbound" || ctx.email?.direction == "internal" - append: + tag: append_user_id_e8d4a6ab field: user.id value: '{{{_temp_.email.to.address}}}' allow_duplicates: false if: ctx._temp_?.email?.to?.address != null && ctx._temp_.email.to.address != "" && ctx.email?.direction == "inbound" || ctx.email?.direction == "internal" - append: + tag: append_user_id_5447623d field: user.id value: '{{{_temp_.email.from.address}}}' allow_duplicates: false if: ctx._temp_?.email?.from?.address != null && ctx._temp_.email.from.address != "" && ctx.email?.direction == "outbound" || ctx.email?.direction == "internal" - script: + tag: script_331f6684 lang: painless description: Reduce user.* to keyword if a single element. source: | @@ -313,23 +373,28 @@ processors: ctx.user.id = ctx.user.id[0]; } - set: + tag: set_destination_user_name_217a8e6f field: destination.user.name copy_from: _temp_.to_user_name if: ctx._temp_?.to_user_name != null && ctx._temp_.to_user_name != "" - set: + tag: set_destination_user_domain_0f8b221d field: destination.user.domain copy_from: _temp_.to_user_domain if: ctx._temp_?.to_user_domain != null && ctx._temp_.to_user_domain != "" - set: + tag: set_source_user_name_d82b6cf7 field: source.user.name copy_from: _temp_.from_user_name if: ctx._temp_?.from_user_name != null && ctx._temp_.from_user_name != "" - set: + tag: set_source_user_domain_4442508f field: source.user.domain copy_from: _temp_.from_user_domain if: ctx._temp_?.from_user_domain != null && ctx._temp_.from_user_domain != "" # Add unique fingerprint for deduplication - fingerprint: + tag: fingerprint_e1052810 fields: - email.from.address - email.to.address @@ -342,14 +407,17 @@ processors: target_field: "_id" # Cleanup fields - remove: + tag: remove__temp__ef8e4353 field: _temp_ ignore_failure: true ignore_missing: true - remove: + tag: remove__conf_67aec840 field: _conf ignore_failure: true ignore_missing: true - script: + tag: script_a3eb5add description: Drops null/empty values recursively. lang: painless source: | diff --git a/packages/microsoft_exchange_online_message_trace/manifest.yml b/packages/microsoft_exchange_online_message_trace/manifest.yml index 444ae00a20b..78781978ded 100644 --- a/packages/microsoft_exchange_online_message_trace/manifest.yml +++ b/packages/microsoft_exchange_online_message_trace/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.3.2" name: microsoft_exchange_online_message_trace title: "Microsoft Exchange Online Message Trace" -version: "2.2.5" +version: "2.2.6" description: "Microsoft Exchange Online Message Trace Integration" type: integration categories: diff --git a/packages/microsoft_intune/changelog.yml b/packages/microsoft_intune/changelog.yml index 6dba1bdf96b..8000cc7a196 100644 --- a/packages/microsoft_intune/changelog.yml +++ b/packages/microsoft_intune/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.1.1" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: 0.1.0 changes: - description: Initial release of package. diff --git a/packages/microsoft_intune/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/microsoft_intune/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index d7b9a8b1c50..29661632d9a 100644 --- a/packages/microsoft_intune/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/microsoft_intune/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -25,6 +25,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -95,6 +96,7 @@ processors: if: ctx.event?.action != '' on_failure: - append: + tag: append_error_message_ce1e3af4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - join: @@ -104,6 +106,7 @@ processors: if: ctx.event?.action != '' on_failure: - append: + tag: append_error_message_abbd6eec field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -115,6 +118,7 @@ processors: if: ctx.json?.properties?.ActivityDate != null && ctx.json.properties.ActivityDate != '' on_failure: - append: + tag: append_error_message_fdcde279 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -158,6 +162,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e06b289f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -275,6 +280,7 @@ processors: value: '{{{_ingest._value.Name}}}' allow_duplicates: false - foreach: + tag: foreach_json_properties_Targets_e2d78ac5 field: json.properties.Targets if: ctx.json?.properties?.Targets instanceof List processor: @@ -284,6 +290,7 @@ processors: target_field: _ingest._value.name ignore_missing: true - foreach: + tag: foreach_json_properties_Targets_b2fd0cc6 field: json.properties.Targets if: ctx.json?.properties?.Targets instanceof List processor: @@ -297,6 +304,7 @@ processors: target_field: _ingest._value.name ignore_missing: true - foreach: + tag: foreach_json_properties_Targets_d06a3334 field: json.properties.Targets if: ctx.json?.properties?.Targets instanceof List processor: @@ -310,6 +318,7 @@ processors: target_field: _ingest._value.old ignore_missing: true - foreach: + tag: foreach_json_properties_Targets_df99a493 field: json.properties.Targets if: ctx.json?.properties?.Targets instanceof List processor: @@ -323,6 +332,7 @@ processors: target_field: _ingest._value.new ignore_missing: true - foreach: + tag: foreach_json_properties_Targets_42288209 field: json.properties.Targets if: ctx.json?.properties?.Targets instanceof List processor: @@ -394,6 +404,7 @@ processors: if: ctx.json?.time != null && ctx.json.time != '' on_failure: - append: + tag: append_error_message_5f36ee57 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: diff --git a/packages/microsoft_intune/data_stream/managed_device/elasticsearch/ingest_pipeline/default.yml b/packages/microsoft_intune/data_stream/managed_device/elasticsearch/ingest_pipeline/default.yml index f7a4f83be0b..52689a21513 100644 --- a/packages/microsoft_intune/data_stream/managed_device/elasticsearch/ingest_pipeline/default.yml +++ b/packages/microsoft_intune/data_stream/managed_device/elasticsearch/ingest_pipeline/default.yml @@ -25,6 +25,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -83,6 +84,7 @@ processors: if: ctx.json?.properties?.CreatedDate != null && ctx.json.properties.CreatedDate != '' on_failure: - append: + tag: append_error_message_354a23e6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -144,6 +146,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_b5531602 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -160,6 +163,7 @@ processors: if: ctx.json?.properties?.InGracePeriodUntil != null && ctx.json.properties.InGracePeriodUntil != '' on_failure: - append: + tag: append_error_message_550f8b20 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -186,6 +190,7 @@ processors: if: ctx.json?.properties?.LastContact != null && ctx.json.properties.LastContact != '' on_failure: - append: + tag: append_error_message_be478d24 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -292,6 +297,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_7215b376 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -302,6 +308,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_dda6219a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -383,6 +390,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_6c478c47 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - uppercase: @@ -391,6 +399,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f58efe9c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -426,6 +435,7 @@ processors: if: ctx.event?.action != '' on_failure: - append: + tag: append_error_message_ce1e3af4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - join: @@ -435,6 +445,7 @@ processors: if: ctx.event?.action != '' on_failure: - append: + tag: append_error_message_abbd6eec field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -461,6 +472,7 @@ processors: if: ctx.json?.time != null && ctx.json.time != '' on_failure: - append: + tag: append_error_message_7aebc872 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: diff --git a/packages/microsoft_intune/manifest.yml b/packages/microsoft_intune/manifest.yml index 17541140db3..372e36083b1 100644 --- a/packages/microsoft_intune/manifest.yml +++ b/packages/microsoft_intune/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.5.4 name: microsoft_intune title: Microsoft Intune -version: 0.1.0 +version: 0.1.1 description: Collect logs from Microsoft Intune with Elastic Agent. type: integration categories: diff --git a/packages/microsoft_sentinel/changelog.yml b/packages/microsoft_sentinel/changelog.yml index 6cd1d0cc62f..015e835088e 100644 --- a/packages/microsoft_sentinel/changelog.yml +++ b/packages/microsoft_sentinel/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.4.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.4.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/microsoft_sentinel/data_stream/alert/elasticsearch/ingest_pipeline/default.yml b/packages/microsoft_sentinel/data_stream/alert/elasticsearch/ingest_pipeline/default.yml index 5d6c3a4f88a..ea33a294703 100644 --- a/packages/microsoft_sentinel/data_stream/alert/elasticsearch/ingest_pipeline/default.yml +++ b/packages/microsoft_sentinel/data_stream/alert/elasticsearch/ingest_pipeline/default.yml @@ -10,7 +10,8 @@ processors: tag: set_event_kind_alert value: alert - remove: - field: + tag: remove_8f6a9bf0 + field: - oberver.product - oberver.vendor ignore_missing: true @@ -53,6 +54,7 @@ processors: if: ctx.event?.original != null on_failure: - append: + tag: append_error_message_04014d20 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - fingerprint: @@ -152,6 +154,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_fc05ea48 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -178,6 +181,7 @@ processors: if: ctx.json?.properties?.endTimeUtc != null && ctx.json.properties.endTimeUtc != '' on_failure: - append: + tag: append_error_message_e03b1c37 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -204,6 +208,7 @@ processors: if: ctx.json?.properties?.processingEndTime != null && ctx.json.properties.processingEndTime != '' on_failure: - append: + tag: append_error_message_acf21bfc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -274,6 +279,7 @@ processors: } on_failure: - append: + tag: append_error_message_c41d93cc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -285,6 +291,7 @@ processors: if: ctx.json?.properties?.startTimeUtc != null && ctx.json.properties.startTimeUtc != '' on_failure: - append: + tag: append_error_message_436eb920 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -303,6 +310,7 @@ processors: ctx.event['duration'] = ChronoUnit.NANOS.between(event_start, event_end); on_failure: - append: + tag: append_error_message_16889e98 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -321,6 +329,7 @@ processors: target_field: microsoft_sentinel.alert.properties.tactics ignore_missing: true - foreach: + tag: foreach_microsoft_sentinel_alert_properties_tactics_33144fe0 field: microsoft_sentinel.alert.properties.tactics if: ctx.microsoft_sentinel?.alert?.properties?.tactics instanceof List processor: @@ -338,6 +347,7 @@ processors: if: ctx.json?.properties?.timeGenerated != null && ctx.json.properties.timeGenerated != '' on_failure: - append: + tag: append_error_message_816df8e4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -359,6 +369,7 @@ processors: if: ctx.json?.systemData?.createdAt != null && ctx.json.systemData.createdAt != '' on_failure: - append: + tag: append_error_message_f7fc51e8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -392,6 +403,7 @@ processors: if: ctx.json?.systemData?.lastModifiedAt != null && ctx.json.systemData.lastModifiedAt != '' on_failure: - append: + tag: append_error_message_aa229d20 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -406,7 +418,7 @@ processors: ignore_missing: true - append: field: related.user - tag: append_user_id_into_related_user + tag: append_related_user_3562a589 value: '{{{microsoft_sentinel.alert.system_data.last_modified_by}}}' allow_duplicates: false if: ctx.microsoft_sentinel?.alert?.system_data?.last_modified_by != null && ctx.microsoft_sentinel?.alert?.system_data?.last_modified_by_type?.toLowerCase() == "user" @@ -455,6 +467,7 @@ processors: description: Drops null/empty values recursively. on_failure: - append: + tag: append_error_message_0886867f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -463,6 +476,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/microsoft_sentinel/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/microsoft_sentinel/data_stream/event/elasticsearch/ingest_pipeline/default.yml index b0b2d84eee4..31f49bac665 100644 --- a/packages/microsoft_sentinel/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/microsoft_sentinel/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -43,6 +43,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - fingerprint: @@ -104,6 +105,7 @@ processors: if: ctx.json?.ConfidenceScore != null && ctx.json.ConfidenceScore != '' on_failure: - append: + tag: append_error_message_439c9e6c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -130,6 +132,7 @@ processors: if: ctx.json?.EndTime != null && ctx.json.EndTime != '' on_failure: - append: + tag: append_error_message_9d8bb9b3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -144,6 +147,7 @@ processors: if: ctx.json?.Entities instanceof String && ctx.json.Entities != '' on_failure: - append: + tag: append_error_message_d5a8402b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -163,6 +167,7 @@ processors: if: ctx.json?.ExtendedProperties instanceof String && ctx.json.ExtendedProperties != '' on_failure: - append: + tag: append_error_message_d51339b6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -171,6 +176,7 @@ processors: target_field: microsoft_sentinel.event.internal_workspace_resource_id ignore_missing: true - set: + tag: set_json_IsIncident_02193af4 field: json.IsIncident value: false if: ctx.json?.IsIncident != null && ctx.json.IsIncident == 0 @@ -182,6 +188,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_6a2208a6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -198,6 +205,7 @@ processors: if: ctx.json?.ProcessingEndTime != null && ctx.json.ProcessingEndTime != '' on_failure: - append: + tag: append_error_message_41a36adf field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -270,6 +278,7 @@ processors: if: ctx.json?.StartTime != null && ctx.json.StartTime != '' on_failure: - append: + tag: append_error_message_2f59e976 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -288,6 +297,7 @@ processors: ctx.event['duration'] = ChronoUnit.NANOS.between(event_start, event_end); on_failure: - append: + tag: append_error_message_16889e98 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -318,9 +328,11 @@ processors: if: ctx.microsoft_sentinel?.event?.tactics instanceof String on_failure: - append: + tag: append_error_message_5606b29a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_microsoft_sentinel_event_tactics_a5c7eb29 field: microsoft_sentinel.event.tactics if: ctx.microsoft_sentinel?.event?.tactics instanceof List processor: @@ -342,9 +354,11 @@ processors: if: ctx.microsoft_sentinel?.event?.techniques instanceof String on_failure: - append: + tag: append_error_message_21662d06 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_microsoft_sentinel_event_techniques_fd279adb field: microsoft_sentinel.event.techniques if: ctx.microsoft_sentinel?.event?.techniques instanceof List processor: @@ -372,6 +386,7 @@ processors: if: ctx.json?.TimeGenerated != null && ctx.json.TimeGenerated != '' on_failure: - append: + tag: append_error_message_4d5ce763 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -458,6 +473,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/microsoft_sentinel/data_stream/incident/elasticsearch/ingest_pipeline/default.yml b/packages/microsoft_sentinel/data_stream/incident/elasticsearch/ingest_pipeline/default.yml index 9490eb6d8da..e0f29e92486 100644 --- a/packages/microsoft_sentinel/data_stream/incident/elasticsearch/ingest_pipeline/default.yml +++ b/packages/microsoft_sentinel/data_stream/incident/elasticsearch/ingest_pipeline/default.yml @@ -40,6 +40,7 @@ processors: if: ctx.event?.original != null on_failure: - append: + tag: append_error_message_04014d20 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - fingerprint: @@ -83,6 +84,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a7b8eda7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -94,6 +96,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e7fc29b4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -105,6 +108,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_508b559a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -118,6 +122,7 @@ processors: target_field: microsoft_sentinel.incident.properties.additional_data.tactics ignore_missing: true - foreach: + tag: foreach_microsoft_sentinel_incident_properties_additional_data_tactics_f72b205a field: microsoft_sentinel.incident.properties.additional_data.tactics if: ctx.microsoft_sentinel?.incident?.properties?.additional_data?.tactics instanceof List processor: @@ -150,6 +155,7 @@ processors: if: ctx.json?.properties?.createdTimeUtc != null && ctx.json.properties.createdTimeUtc != '' on_failure: - append: + tag: append_error_message_37f45614 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -176,6 +182,7 @@ processors: if: ctx.json?.properties?.firstActivityTimeUtc != null && ctx.json.properties.firstActivityTimeUtc != '' on_failure: - append: + tag: append_error_message_7f43791e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -187,6 +194,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_7400c2d2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -218,6 +226,7 @@ processors: if: ctx.json?.properties?.lastActivityTimeUtc != null && ctx.json.properties.lastActivityTimeUtc != '' on_failure: - append: + tag: append_error_message_044e1f72 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -229,6 +238,7 @@ processors: if: ctx.json?.properties?.lastModifiedTimeUtc != null && ctx.json.properties.lastModifiedTimeUtc != '' on_failure: - append: + tag: append_error_message_d4289ae8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -316,6 +326,7 @@ processors: target_field: microsoft_sentinel.incident.properties.related_analytic_rule_ids ignore_missing: true - foreach: + tag: foreach_microsoft_sentinel_incident_properties_related_analytic_rule_ids_9ffee0a7 field: microsoft_sentinel.incident.properties.related_analytic_rule_ids if: ctx.microsoft_sentinel?.incident?.properties?.related_analytic_rule_ids instanceof List processor: @@ -347,6 +358,7 @@ processors: } on_failure: - append: + tag: append_error_message_376d66ef field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -368,6 +380,7 @@ processors: if: ctx.json?.systemData?.createdAt != null && ctx.json.systemData.createdAt != '' on_failure: - append: + tag: append_error_message_01a08f98 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -395,6 +408,7 @@ processors: if: ctx.json?.systemData?.lastModifiedAt != null && ctx.json.systemData.lastModifiedAt != '' on_failure: - append: + tag: append_error_message_2e0c05fc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -462,6 +476,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/microsoft_sentinel/manifest.yml b/packages/microsoft_sentinel/manifest.yml index 96feb906bab..097916074a8 100644 --- a/packages/microsoft_sentinel/manifest.yml +++ b/packages/microsoft_sentinel/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: microsoft_sentinel title: Microsoft Sentinel -version: "1.4.1" +version: "1.4.2" description: Collect logs from Microsoft Sentinel with Elastic Agent. type: integration categories: diff --git a/packages/mimecast/changelog.yml b/packages/mimecast/changelog.yml index ff48b173985..5d35dd9e48a 100644 --- a/packages/mimecast/changelog.yml +++ b/packages/mimecast/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.5.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "3.5.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/mimecast/data_stream/archive_search_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/archive_search_logs/elasticsearch/ingest_pipeline/default.yml index c21655459b0..6a9c7ee200e 100644 --- a/packages/mimecast/data_stream/archive_search_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/archive_search_logs/elasticsearch/ingest_pipeline/default.yml @@ -3,6 +3,7 @@ description: Pipeline for processing Mimecast archive_search_logs. processors: # Generic event/ecs fields we always want to populate. - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - remove: @@ -30,18 +31,22 @@ processors: description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null - json: + tag: json_event_original_to_mimecast_d72eb09c description: Parse 'message' JSON contents into a 'mimecast' object. field: event.original target_field: mimecast - drop: + tag: drop_752a620f if: ctx.mimecast?.createTime == null || (ctx.mimecast?.data != null && ctx.mimecast.data.isEmpty()) - fingerprint: + tag: fingerprint_b0c83cf3 fields: - mimecast.id target_field: "_id" ignore_missing: true - date: + tag: date_mimecast_createTime_9e9563ec description: Use 'mimecast.createTime' as the '@timestamp' field: mimecast.createTime timezone: UTC @@ -62,22 +67,27 @@ processors: # Search related fields - rename: + tag: rename_mimecast_searchText_to_mimecast_search_details_text_60a182f7 field: mimecast.searchText target_field: mimecast.search_details.text ignore_missing: true - rename: + tag: rename_mimecast_description_to_mimecast_search_details_description_e0295549 field: mimecast.description target_field: mimecast.search_details.description ignore_missing: true - rename: + tag: rename_mimecast_source_to_mimecast_search_details_source_6016c9db field: mimecast.source target_field: mimecast.search_details.source ignore_missing: true - rename: + tag: rename_mimecast_searchPath_to_mimecast_search_details_path_3ecd656f field: mimecast.searchPath target_field: mimecast.search_details.path ignore_missing: true - rename: + tag: rename_mimecast_searchReason_to_mimecast_search_details_reason_4867c79b field: mimecast.searchReason target_field: mimecast.search_details.reason ignore_missing: true @@ -86,54 +96,66 @@ processors: # User fields - remove: + tag: remove_mimecast_emailAddr_55db77c8 field: mimecast.emailAddr if: ctx.mimecast?.emailAddr == "<>" - rename: + tag: rename_mimecast_emailAddr_to_user_email_b6f2ff29 field: mimecast.emailAddr target_field: user.email ignore_missing: true - dissect: + tag: dissect_user_email_00f1db91 field: user.email pattern: "<%{user.email}>" ignore_missing: true ignore_failure: true - split: + tag: split_user_email_to_user_parts_5e89f261 field: user.email separator: "@" target_field: user.parts if: 'ctx.user?.email != null' - set: + tag: set_user_name_607743de field: user.name copy_from: user.parts.0 if: ctx.user?.parts?.length != null && ctx.user.parts.length > 1 - set: + tag: set_user_domain_49f3b1be field: user.domain copy_from: user.parts.1 if: ctx.user?.parts?.length != null && ctx.user.parts.length > 1 - + - append: + tag: append_related_user_039a5a91 field: related.user value: "{{{user.name}}}" allow_duplicates: false if: 'ctx.user?.name !=null' - append: + tag: append_related_user_34fcf415 field: related.user value: "{{{user.email}}}" allow_duplicates: false if: ctx.user?.email != null - + # Event related fields # Sets event.created from the @timestamp field generated by filebeat before being overwritten further down - set: + tag: set_event_created_e3f09e3b field: event.created copy_from: "@timestamp" - set: + tag: set_event_kind_de80643c field: event.kind value: event - append: + tag: append_event_category_4520c8f4 field: event.category value: [api] - append: + tag: append_event_type_9d7bc353 field: event.type value: [admin] @@ -166,8 +188,9 @@ processors: } handleMap(ctx); - remove: + tag: remove_4b706cc6 description: Cleanup of repeated/unwanted/temporary fields. - field: + field: - mimecast.createTime - mimecast.searchPath - mimecast.searchText diff --git a/packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml index f5eccebc78c..702254e1be2 100644 --- a/packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml @@ -3,6 +3,7 @@ description: Pipeline for processing Mimecast audit_events. processors: # Generic event/ecs fields we always want to populate. - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - remove: @@ -30,21 +31,26 @@ processors: description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null - json: + tag: json_event_original_to_mimecast_d72eb09c description: Parse 'message' JSON contents into a 'mimecast' object. field: event.original target_field: mimecast - drop: + tag: drop_dd4ed123 if: ctx.mimecast?.eventTime == null || (ctx.mimecast?.data != null && ctx.mimecast.data.isEmpty()) - fingerprint: + tag: fingerprint_b0c83cf3 fields: - mimecast.id target_field: "_id" ignore_missing: true - set: + tag: set_event_category_4861796c field: event.category value: [email] - date: + tag: date_mimecast_eventTime_e007390c description: Use 'mimecast.eventTime' as the '@timestamp' field: mimecast.eventTime timezone: UTC @@ -66,14 +72,17 @@ processors: # Convert 'mimecast.auditType' to a bone-cased event action. # ie: User Log On -> user-log-on - lowercase: + tag: lowercase_mimecast_auditType_2cd969a3 field: mimecast.auditType ignore_missing: true - gsub: + tag: gsub_mimecast_auditType_ba5dc1e8 field: mimecast.auditType pattern: " " replacement: "-" ignore_missing: true - rename: + tag: rename_mimecast_auditType_to_event_action_e9c9153e field: mimecast.auditType target_field: event.action ignore_missing: true @@ -81,19 +90,21 @@ processors: # User fields - rename: + tag: rename_mimecast_user_to_user_email_a1ca6321 field: mimecast.user target_field: user.email ignore_missing: true - rename: + tag: rename_mimecast_id_to_event_id_c7737d91 field: mimecast.id target_field: event.id ignore_missing: true + ### # Here we want to add as much categorization information as possible # We can do this by parsing mimecast.eventInfo differently based on # what event.action is, etc. ### - - grok: tag: grok_logon_authentication_failed field: mimecast.eventInfo @@ -108,24 +119,28 @@ processors: TIMEZONE: '(?:[A-Z]{3,4}|(?:GMT)?[-+][0-9]{2}:?[0-9]{2})' ignore_missing: true - dissect: + tag: dissect_mimecast_eventInfo_9234862f field: mimecast.eventInfo pattern: "%{mimecast.info}>, %{mimecast.rest_of_event_info}" if: 'ctx.mimecast?.info == null' ignore_missing: true ignore_failure: true - dissect: + tag: dissect_mimecast_eventInfo_edaca02e field: mimecast.eventInfo pattern: "%{mimecast.info}, %{mimecast.rest_of_event_info}" if: 'ctx.mimecast?.info == null' ignore_missing: true ignore_failure: true - dissect: + tag: dissect_mimecast_eventInfo_3c259f1f field: mimecast.eventInfo pattern: "%{?drop->} - %{mimecast.info}<%{user.email}> %{?key}: %{mimecast.date} %{?key}: %{mimecast.time} %{mimecast.timezone} %{?key}: %{client.ip} %{?key}: %{mimecast.application}" if: 'ctx.event?.action=="folder-log-entry" || ctx.event?.action=="custom-report-definition-created" || ctx.event?.action=="mimecast-support-login"' ignore_missing: true ignore_failure: true - kv: + tag: kv_mimecast_rest_of_event_info_to_mimecast_event_info_parts_24e368e1 field: mimecast.rest_of_event_info field_split: ", " value_split: ": " @@ -133,6 +148,7 @@ processors: ignore_failure: true ignore_missing: true - set: + tag: set_mimecast_remote_4f408690 field: mimecast.remote value: "{{{mimecast.rest_of_event_info}}}" if: 'ctx.event?.action=="user-logged-on" && ctx?.mimecast?.event_info_parts?.IP == null' @@ -144,64 +160,78 @@ processors: ignore_missing: true ignore_failure: true - rename: + tag: rename_mimecast_event_info_parts_Date_to_mimecast_date_08cb1db2 field: mimecast.event_info_parts.Date target_field: mimecast.date ignore_missing: true - dissect: + tag: dissect_mimecast_event_info_parts_Time_8303363c field: mimecast.event_info_parts.Time pattern: '%{mimecast.event_info_parts.Time} %{mimecast.timezone}' ignore_missing: true ignore_failure: true - rename: + tag: rename_mimecast_event_info_parts_Time_to_mimecast_time_74b26af8 field: mimecast.event_info_parts.Time target_field: mimecast.time ignore_missing: true - rename: + tag: rename_mimecast_event_info_parts_IP_to_client_ip_5c0b0ad0 field: mimecast.event_info_parts.IP target_field: client.ip ignore_missing: true - rename: + tag: rename_mimecast_event_info_parts_Application_to_mimecast_application_59a0a966 field: mimecast.event_info_parts.Application target_field: mimecast.application ignore_missing: true - rename: + tag: rename_mimecast_event_info_parts_Method_to_mimecast_method_689f5da8 field: mimecast.event_info_parts.Method target_field: mimecast.method ignore_missing: true - rename: + tag: rename_mimecast_event_info_parts_Reason_to_event_reason_184c4bcf field: mimecast.event_info_parts.Reason target_field: event.reason ignore_missing: true - - rename: + - rename: + tag: rename_mimecast_info_to_mimecast_filename_938ce24b field: mimecast.info target_field: mimecast.filename ignore_missing: true if: 'ctx.event?.action == "threat-intel-feed-download"' - - rename: + - rename: + tag: rename_mimecast_event_info_parts_Processed_to_email_origination_timestamp_23e5e7b5 field: mimecast.event_info_parts.Processed target_field: email.origination_timestamp ignore_missing: true - - rename: + - rename: + tag: rename_mimecast_event_info_parts_Subject_to_email_subject_dd821675 field: mimecast.event_info_parts.Subject target_field: email.subject ignore_missing: true - - rename: + - rename: + tag: rename_mimecast_event_info_parts_2FA_to_mimecast_2FA_8cdc043e field: mimecast.event_info_parts.2FA target_field: mimecast.2FA ignore_missing: true - dissect: - field: mimecast.event_info_parts.From + tag: dissect_mimecast_event_info_parts_From_58719332 + field: mimecast.event_info_parts.From pattern: "<%{?drop}> %{email.from.address}" if: 'ctx.event?.action=="message-action"' ignore_missing: true ignore_failure: true - dissect: + tag: dissect_mimecast_event_info_parts_To_28455ef4 field: mimecast.event_info_parts.To pattern: "<%{?drop}> %{email.to.address}" if: 'ctx.event?.action=="message-action"' ignore_missing: true - ignore_failure: true + ignore_failure: true - dissect: + tag: dissect_mimecast_eventInfo_8e88f4d5 field: mimecast.eventInfo pattern: "[%{?key} : %{mimecast.export_type},%{?key} :%{mimecast.export_name},%{?key} :%{user.email},%{?key} :%{mimecast.weekday} %{mimecast.month} %{mimecast.monthday} %{mimecast.time} %{mimecast.timezone} %{mimecast.year},%{?key} :%{client.ip},%{?key} :%{mimecast.columns_exported},%{?key} : %{file.name},%{?key}: %{file.size},%{?key} : %{file.extension}], %{?key}: %{mimecast.date}, %{?key}: %{mimecast.time}, %{?key}: %{client.ip}, %{?key}: %{mimecast.application}" if: 'ctx.event?.action=="page-data-exports"' @@ -216,56 +246,68 @@ processors: ignore_failure: true if: 'ctx?.event?.action=="user-logged-on" && ctx?.mimecast?.event_info_parts?.IP == null' - set: + tag: set_email_from_address_0f71febc field: email.from.address value: ["{{{email.from.address}}}"] if: "ctx?.email?.from?.address != null" - set: + tag: set_email_to_address_f0f0f9f3 field: email.to.address value: ["{{{email.to.address}}}"] if: "ctx?.email?.to?.address != null" - dissect: + tag: dissect_email_from_address_969e2d4f field: email.from.address pattern: "<%{email.from.address}>" ignore_missing: true ignore_failure: true - dissect: + tag: dissect_email_to_address_3f03f6e9 field: email.to.address pattern: "<%{email.to.address}>" ignore_missing: true - ignore_failure: true + ignore_failure: true - convert: + tag: convert_file_size_02653c50 field: file.size type: long ignore_missing: true - split: + tag: split_user_email_to_user_parts_07aa1616 field: user.email separator: "@" target_field: user.parts if: 'ctx?.user?.email != null' - set: + tag: set_user_name_8dbd753e field: user.name copy_from: user.parts.0 if: 'ctx?.user?.parts !=null && ctx?.user?.parts.length > 1' - set: + tag: set_user_domain_11fbe89e field: user.domain copy_from: user.parts.1 if: 'ctx?.user?.parts !=null && ctx?.user?.parts.length > 1' - - rename: + - rename: + tag: rename_mimecast_filename_to_file_name_5ee6418c field: mimecast.filename target_field: file.name ignore_missing: true if: 'ctx?.mimecast?.filename != null && ctx?.event?.action == "threat-intel-feed-download"' - split: + tag: split_file_name_to_file_parts_b6308be5 field: file.name separator: "\\." target_field: file.parts if: 'ctx?.file?.name != null && ctx?.event?.action == "threat-intel-feed-download"' - script: + tag: script_9b793198 lang: painless source: | ctx.file.extension = ctx.file.parts[ctx.file.parts.length-1]; if: 'ctx?.file?.parts !=null && ctx?.file?.parts.length > 1' - set: + tag: set_event_created_a06e6881 field: event.created value: "{{{mimecast.date}}} {{{mimecast.time}}}" if: 'ctx?.mimecast?.date != null && ctx?.mimecast?.time != null' @@ -291,10 +333,12 @@ processors: - yyyy-MM-dd'T'HH:mm:ss z if: ctx.event?.created != null - geoip: + tag: geoip_client_ip_to_client_geo_90831252 field: client.ip target_field: client.geo ignore_missing: true - geoip: + tag: geoip_client_ip_to_client_as_f17fb2b3 database_file: GeoLite2-ASN.mmdb field: client.ip target_field: client.as @@ -303,40 +347,48 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_client_as_asn_to_client_as_number_a6e30d01 field: client.as.asn target_field: client.as.number ignore_missing: true - rename: + tag: rename_client_as_organization_name_to_client_as_organization_name_817a526f field: client.as.organization_name target_field: client.as.organization.name ignore_missing: true - append: + tag: append_related_ip_1bf248e3 field: related.ip value: "{{{client.ip}}}" allow_duplicates: false if: ctx.client?.ip !=null - append: + tag: append_related_ip_a328ed6f field: related.ip value: "{{{mimecast.remote_ip}}}" allow_duplicates: false if: ctx.mimecast?.remote_ip !=null - append: + tag: append_related_user_570f5bce field: related.user value: "{{{user.name}}}" allow_duplicates: false if: 'ctx?.user?.name !=null' - append: + tag: append_related_user_a5a03234 field: related.user value: "{{{user.email}}}" allow_duplicates: false if: ctx?.user?.email != null - lowercase: + tag: lowercase_email_direction_569c0270 field: email.direction ignore_missing: true # Cleanup - remove: + tag: remove_81a9e570 description: Cleanup of repeated/unwanted/temporary fields. - field: + field: - mimecast.eventTime - user.parts - mimecast.date @@ -392,6 +444,7 @@ processors: } handleMap(ctx); + ### # Error handling diff --git a/packages/mimecast/data_stream/cloud_integrated_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/cloud_integrated_logs/elasticsearch/ingest_pipeline/default.yml index 3827c458aaf..7ecaf79dd86 100644 --- a/packages/mimecast/data_stream/cloud_integrated_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/cloud_integrated_logs/elasticsearch/ingest_pipeline/default.yml @@ -2,10 +2,12 @@ description: Pipeline for processing Mimecast cloud_integrated_logs. processors: - drop: + tag: drop_d6785451 description: Ignore want_more placeholder message. if: ctx.message == "want_more" - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - remove: @@ -33,16 +35,20 @@ processors: description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null - json: + tag: json_event_original_to_mimecast_d72eb09c description: Parse 'message' JSON contents into a 'mimecast' object. field: event.original target_field: mimecast - drop: + tag: drop_6459310c if: ctx?.mimecast?.datetime == null && ctx?.mimecast?.timestamp == null - set: + tag: set_event_category_4861796c field: event.category value: [email] - date: + tag: date_mimecast_timestamp_2f9b8186 description: Use 'mimecast.timestamp' as the '@timestamp' field: mimecast.timestamp if: ctx.mimecast?.timestamp != null @@ -50,16 +56,19 @@ processors: formats: - UNIX_MS - set: + tag: set_event_created_3b69f632 field: event.created copy_from: '@timestamp' if: ctx['@timestamp'] != null - set: + tag: set_event_kind_6ef68e78 field: event.kind value: alert if: ctx.mimecast?.tags instanceof List && ctx.mimecast.tags.length != 0 ### NOTE LOG TYPE - rename: + tag: rename_mimecast_type_to_mimecast_log_type_8751ca79 field: mimecast.type target_field: mimecast.log_type ignore_missing: true @@ -72,6 +81,7 @@ processors: if: ctx.tags?.contains('preserve_duplicate_custom_fields') == true - fingerprint: + tag: fingerprint_5cae5d9e fields: - mimecast.messageId - mimecast.processingId @@ -86,6 +96,7 @@ processors: ### Common - foreach: + tag: foreach_mimecast_recipients_9bfe67b6 if: ctx.mimecast?.recipients instanceof List field: mimecast.recipients processor: @@ -94,6 +105,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - append: + tag: append_email_to_address_0cfb7a4a field: email.to.address value: '{{{mimecast.recipients}}}' allow_duplicates: false @@ -101,39 +113,48 @@ processors: ### ENTITIES - rename: + tag: rename_mimecast_senderIp_to_source_ip_e17dfbc5 field: mimecast.senderIp target_field: source.ip ignore_missing: true - rename: + tag: rename_mimecast_messageId_to_email_message_id_a8ab0b64 field: mimecast.messageId target_field: email.message_id ignore_missing: true - append: + tag: append_email_from_address_0cbef6e5 field: email.from.address value: '{{{mimecast.senderHeader}}}' allow_duplicates: false if: ctx.mimecast?.senderHeader != null - append: + tag: append_email_from_address_3477ed67 field: email.from.address value: '{{{mimecast.senderEnvelope}}}' allow_duplicates: false if: ctx.mimecast?.senderEnvelope != null - rename: + tag: rename_mimecast_subject_to_email_subject_3c7206f3 field: mimecast.subject target_field: email.subject ignore_missing: true - rename: + tag: rename_mimecast_direction_to_email_direction_624c8c4d field: mimecast.direction target_field: email.direction ignore_missing: true - lowercase: + tag: lowercase_email_direction_569c0270 field: email.direction ignore_missing: true - rename: + tag: rename_mimecast_attachments_to_email_attachments_file_name_e8d63fdc field: mimecast.attachments target_field: email.attachments.file.name ignore_missing: true - rename: + tag: rename_mimecast_action_to_mimecast_threatState_99481815 field: mimecast.action target_field: mimecast.threatState ignore_missing: true @@ -143,33 +164,40 @@ processors: ### URL LOGS - rename: + tag: rename_mimecast_action_to_event_action_75b859cd field: mimecast.action target_field: event.action ignore_missing: true - rename: + tag: rename_mimecast_sourceIp_to_source_ip_a8821cb9 field: mimecast.sourceIp target_field: source.ip ignore_missing: true - convert: + tag: convert_source_ip_51566224 field: source.ip type: ip ignore_missing: true on_failure: - append: + tag: append_error_message_35cc2af9 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' - remove: + tag: remove_source_ip_07943d1d field: source.ip ignore_failure: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -178,15 +206,18 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - foreach: + tag: foreach_email_from_address_bd2fca99 field: email.from.address if: ctx.email?.from?.address instanceof List ignore_failure: true @@ -197,6 +228,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - foreach: + tag: foreach_email_to_address_5d15e64f field: email.to.address if: ctx.email?.to?.address instanceof List ignore_failure: true @@ -248,11 +280,13 @@ processors: # Cleanup - rename: + tag: rename_original_to_mimecast_5b6791aa field: original target_field: mimecast override: true ignore_missing: true - remove: + tag: remove_0a5321eb description: Cleanup of repeated/unwanted/temporary fields. field: - mimecast._offset diff --git a/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml index 0ba9cbd5119..a93c4faed95 100644 --- a/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml @@ -3,6 +3,7 @@ description: Pipeline for processing Mimecast dlp_logs. processors: # Generic event/ecs fields we always want to populate. - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - remove: @@ -30,12 +31,15 @@ processors: description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null - json: + tag: json_event_original_to_mimecast_d72eb09c description: Parse 'message' JSON contents into a 'mimecast' object. field: event.original target_field: mimecast - drop: + tag: drop_dd4ed123 if: ctx.mimecast?.eventTime == null || (ctx.mimecast?.data != null && ctx.mimecast.data.isEmpty()) - fingerprint: + tag: fingerprint_45ad06db fields: - mimecast.messageId - mimecast.action @@ -43,14 +47,17 @@ processors: target_field: "_id" ignore_missing: true - set: + tag: set_event_category_4861796c field: event.category value: [email] - set: + tag: set_event_kind_0ab7b65f field: event.kind value: alert if: ctx._conf?.alerting instanceof List && ctx._conf.alerting.contains(ctx.mimecast?.action) - date: + tag: date_mimecast_eventTime_a835a222 description: Use 'mimecast.eventTime' as the '@timestamp' field: mimecast.eventTime timezone: UTC @@ -59,71 +66,85 @@ processors: ### - append: + tag: append_email_from_address_d80318fb field: email.from.address value: "{{{mimecast.senderAddress}}}" if: "ctx?.mimecast?.senderAddress != null" - append: + tag: append_email_to_address_281f192c field: email.to.address value: "{{{mimecast.recipientAddress}}}" if: "ctx?.mimecast?.recipientAddress != null" - dissect: + tag: dissect_email_from_address_969e2d4f field: email.from.address pattern: "<%{email.from.address}>" ignore_missing: true ignore_failure: true - gsub: + tag: gsub_email_from_address_6a1fe514 field: email.from.address pattern: '<>' replacement: '' ignore_missing: true ignore_failure: true - dissect: + tag: dissect_email_to_address_3f03f6e9 field: email.to.address pattern: "<%{email.to.address}>" ignore_missing: true - ignore_failure: true + ignore_failure: true - gsub: + tag: gsub_email_to_address_1726742b field: email.to.address pattern: '<>' replacement: '' ignore_missing: true ignore_failure: true - rename: + tag: rename_mimecast_action_to_event_action_187d1a3d field: mimecast.action target_field: event.action ignore_missing: true if: 'ctx?.mimecast?.action !=null' - rename: + tag: rename_mimecast_subject_to_email_subject_f6b733ff field: mimecast.subject target_field: email.subject ignore_missing: true if: 'ctx?.mimecast?.subject !=null' - rename: + tag: rename_mimecast_messageId_to_email_message_id_eca64d12 field: mimecast.messageId target_field: email.message_id ignore_missing: true if: 'ctx?.mimecast?.messageId !=null' - rename: + tag: rename_mimecast_route_to_email_direction_880aaa48 field: mimecast.route target_field: email.direction ignore_missing: true if: 'ctx?.mimecast?.route !=null' - rename: + tag: rename_mimecast_policy_to_rule_name_8cd791ba field: mimecast.policy target_field: rule.name ignore_missing: true if: 'ctx?.mimecast?.policy !=null' - set: + tag: set_event_created_6dbfe9df field: event.created value: "{{{mimecast.eventTime}}}" if: 'ctx?.mimecast?.eventTime != null' - lowercase: + tag: lowercase_email_direction_569c0270 field: email.direction ignore_missing: true # Cleanup - remove: + tag: remove_f8e318fb description: Cleanup of repeated/unwanted/temporary fields. - field: + field: - mimecast - _conf ignore_missing: true @@ -156,6 +177,7 @@ processors: } handleMap(ctx); + # Error handling on_failure: - set: diff --git a/packages/mimecast/data_stream/message_release_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/message_release_logs/elasticsearch/ingest_pipeline/default.yml index d74e97bb9f9..9a67ada871b 100644 --- a/packages/mimecast/data_stream/message_release_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/message_release_logs/elasticsearch/ingest_pipeline/default.yml @@ -3,6 +3,7 @@ description: Pipeline for processing Mimecast message_release_logs. processors: # Generic event/ecs fields we always want to populate. - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - remove: @@ -34,6 +35,7 @@ processors: description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null - json: + tag: json_event_original_to_mimecast_d72eb09c description: Parse 'message' JSON contents into a 'mimecast' object. field: event.original target_field: mimecast @@ -48,6 +50,7 @@ processors: - yyyy-MM-dd'T'HH:mm:ssZ - yyyy-MM-dd'T'HH:mm:ssZZZZZ - fingerprint: + tag: fingerprint_8a071f71 fields: - mimecast.id - '@timestamp' @@ -55,16 +58,19 @@ processors: ignore_missing: true - append: + tag: append_email_from_address_455ce224 field: email.from.address value: '{{{mimecast.fromEnv.emailAddress}}}' if: ctx.mimecast?.fromEnv?.emailAddress != null allow_duplicates: false - append: + tag: append_email_from_address_67cf7828 field: email.from.address value: '{{{mimecast.fromHdr.emailAddress}}}' if: ctx.mimecast?.fromHdr?.emailAddress != null allow_duplicates: false - foreach: + tag: foreach_mimecast_to_6747f41e field: mimecast.to if: ctx.mimecast?.to instanceof List processor: @@ -73,33 +79,40 @@ processors: value: '{{{_ingest._value.emailAddress}}}' allow_duplicates: false - set: + tag: set_email_direction_262ca9a0 field: email.direction copy_from: mimecast.route ignore_empty_value: true - set: + tag: set_email_local_id_6e6f096a field: email.local_id copy_from: mimecast.id ignore_empty_value: true - set: + tag: set_email_subject_016ce77c field: email.subject copy_from: mimecast.subject ignore_empty_value: true - set: + tag: set_user_email_5e868477 field: user.email copy_from: email.from.address ignore_empty_value: true if: ctx.email.direction == 'outbound' - append: + tag: append_user_full_name_19316996 field: user.full_name value: '{{{mimecast.fromEnv.displayableName}}}' if: ctx.email.direction == 'outbound' && ctx.mimecast?.fromEnv?.emailAddress != null - set: + tag: set_user_email_7e7fdd9b field: user.email copy_from: email.to.address ignore_empty_value: true if: ctx.email.direction == 'inbound' - foreach: + tag: foreach_mimecast_to_5061dc77 field: mimecast.to if: ctx.email.direction == 'inbound' && ctx.mimecast?.to instanceof List processor: @@ -109,6 +122,7 @@ processors: allow_duplicates: false - script: + tag: script_b998598b description: Populate related.* fields. lang: painless ignore_failure: true @@ -191,37 +205,46 @@ processors: } } - rename: + tag: rename_mimecast_spamProcessingDetail_verdict_to_mimecast_spamProcessingDetail_spamVerdict_3a02e636 description: Move spamProcessingDetail.verdict to legacy location. field: mimecast.spamProcessingDetail.verdict target_field: mimecast.spamProcessingDetail.spamVerdict ignore_missing: true ignore_failure: true - set: + tag: set_event_action_bdd2d6e4 field: event.action copy_from: mimecast.status ignore_empty_value: true - set: + tag: set_event_category_4861796c field: event.category value: [email] - set: + tag: set_event_id_cd8b3bb6 field: event.id copy_from: mimecast.id - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_kind_e5a50a31 field: event.kind value: alert if: ctx.mimecast?.rejectReason != null && ctx.mimecast.rejectReason != '' - set: + tag: set_event_reason_eacf524b field: event.reason copy_from: mimecast.rejectReason ignore_empty_value: true - set: + tag: set_event_risk_score_8cff072f field: event.risk_score copy_from: mimecast.spamScore ignore_empty_value: true - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] diff --git a/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/default.yml index 64cc1047dcc..c59287317f0 100644 --- a/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/default.yml @@ -2,10 +2,12 @@ description: Pipeline for processing Mimecast siem_logs. processors: - drop: + tag: drop_d6785451 description: Ignore want_more placeholder message. if: ctx.message == "want_more" - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - remove: @@ -33,12 +35,15 @@ processors: description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null - json: + tag: json_event_original_to_mimecast_d72eb09c description: Parse 'message' JSON contents into a 'mimecast' object. field: event.original target_field: mimecast - drop: + tag: drop_9635d6e8 if: ctx.mimecast?.datetime == null && ctx.mimecast?.timestamp == null - set: + tag: set_event_category_4861796c field: event.category value: [email] @@ -49,7 +54,7 @@ processors: - pipeline: name: '{{ IngestPipeline "v2_pipeline" }}' if: ctx.mimecast?.timestamp != null - tag: v1_pipeline + tag: pipeline_ccc9936e # Cleanup - script: diff --git a/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/v1_pipeline.yml b/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/v1_pipeline.yml index 15e322db1a6..ed5111b11de 100644 --- a/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/v1_pipeline.yml +++ b/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/v1_pipeline.yml @@ -2,6 +2,7 @@ description: Pipeline for processing Mimecast siem_logs from v1 API. processors: - date: + tag: date_mimecast_datetime_f5f4a578 description: Use 'mimecast.datetime' as the '@timestamp' field: mimecast.datetime if: ctx.mimecast?.datetime != null @@ -11,6 +12,7 @@ processors: ### NOTE LOG TYPE - script: + tag: script_30e15f01 lang: painless if: ctx.mimecast instanceof Map params: @@ -61,7 +63,7 @@ processors: act: [delivery, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, jrnl] attcnt: [receipt, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, jrnl] attsize: [receipt, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, jrnl] - cphr: [process,avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, jrnl] + cphr: [process, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, jrnl] dir: [process, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect] fileext: [receipt, process, delivery, spam, internal-email-protect, impersonation-protect, url-protect, jrnl] filemime: [receipt, process, delivery, spam, internal-email-protect, impersonation-protect, url-protect, jrnl] @@ -179,6 +181,7 @@ processors: ctx.mimecast.log_type = score.keySet(); - fingerprint: + tag: fingerprint_20418e65 fields: - mimecast.MsgId - mimecast.aCode @@ -194,265 +197,326 @@ processors: ### RECEIPT LOGS - rename: + tag: rename_mimecast_aCode_to_email_local_id_23a3a510 field: mimecast.aCode target_field: email.local_id ignore_missing: true - rename: + tag: rename_mimecast_Act_to_event_action_1f58d85f field: mimecast.Act target_field: event.action ignore_missing: true - rename: + tag: rename_mimecast_Cphr_to_tls_cipher_3fb07b42 field: mimecast.Cphr target_field: tls.cipher ignore_missing: true - rename: + tag: rename_mimecast_Dir_to_email_direction_0ad8035b field: mimecast.Dir target_field: email.direction ignore_missing: true - rename: + tag: rename_mimecast_Error_to_error_message_1405b98a field: mimecast.Error target_field: error.message ignore_missing: true - rename: + tag: rename_mimecast_IP_to_source_ip_98860b80 field: mimecast.IP target_field: source.ip ignore_missing: true - rename: + tag: rename_mimecast_MsgId_to_email_message_id_aa9b3f62 field: mimecast.MsgId target_field: email.message_id ignore_missing: true - append: + tag: append_email_to_address_4e6ed654 field: email.to.address value: '{{{mimecast.Rcpt}}}' allow_duplicates: false if: ctx.mimecast?.Rcpt != null - append: + tag: append_email_from_address_fa86979b field: email.from.address value: '{{{mimecast.headerFrom}}}' allow_duplicates: false if: ctx.mimecast?.headerFrom != null - rename: + tag: rename_mimecast_RejCode_to_error_code_325b5730 field: mimecast.RejCode target_field: error.code ignore_missing: true - rename: + tag: rename_mimecast_RejInfo_to_event_reason_8cb28a66 field: mimecast.RejInfo target_field: event.reason ignore_missing: true - set: + tag: set_event_kind_3e1b7681 field: event.kind value: alert if: ctx.mimecast?.RejType != null && ctx.mimecast.RejType != '' - rename: + tag: rename_mimecast_RejType_to_error_type_78686238 field: mimecast.RejType target_field: error.type ignore_missing: true - append: + tag: append_email_from_address_53400426 field: email.from.address value: "{{{mimecast.Sender}}}" allow_duplicates: false if: "ctx?.mimecast?.Sender != null" - rename: + tag: rename_mimecast_Subject_to_email_subject_79b93593 field: mimecast.Subject target_field: email.subject ignore_missing: true - rename: + tag: rename_mimecast_TlsVer_to_tls_version_e90e16ca field: mimecast.TlsVer target_field: tls.version ignore_missing: true ### PROCESS LOGS - rename: + tag: rename_mimecast_AttSize_to_email_attachments_file_size_60fec82e field: mimecast.AttSize target_field: email.attachments.file.size ignore_missing: true - rename: + tag: rename_mimecast_AttNames_to_email_attachments_file_name_fb1a5807 field: mimecast.AttNames target_field: email.attachments.file.name ignore_missing: true - set: + tag: set_event_kind_a51a4ef9 field: event.kind value: alert if: ctx.mimecast?.Hld != null && ctx.mimecast.Hld != '' - rename: + tag: rename_mimecast_Hld_to_event_reason_21dd9fa1 field: mimecast.Hld target_field: event.reason ignore_missing: true ### DELIVERY LOGS - rename: + tag: rename_mimecast_Err_to_error_message_1254a057 field: mimecast.Err target_field: error.message ignore_missing: true - rename: + tag: rename_mimecast_UseTls_to_tls_established_60951eae field: mimecast.UseTls target_field: tls.established ignore_missing: true - set: + tag: set_tls_established_834568c2 field: tls.established value: true if: 'ctx.tls?.established instanceof String && ctx.tls.established.toLowerCase() == "yes"' - set: + tag: set_tls_established_b322f3bd field: tls.established value: false if: 'ctx.tls?.established instanceof String && ctx.tls.established.toLowerCase() == "no"' ### AV LOGS - set: + tag: set_event_kind_7f95ae05 field: event.kind value: alert if: ctx.mimecast?.fileExt != null && ctx.mimecast.fileExt != '' - rename: + tag: rename_mimecast_fileExt_to_email_attachments_file_extension_2348b07d field: mimecast.fileExt target_field: email.attachments.file.extension ignore_missing: true - rename: + tag: rename_mimecast_fileMime_to_email_attachments_file_mime_type_94efc21c field: mimecast.fileMime target_field: email.attachments.file.mime_type ignore_missing: true - rename: + tag: rename_mimecast_md5_to_email_attachments_file_hash_md5_639a4925 field: mimecast.md5 target_field: email.attachments.file.hash.md5 ignore_missing: true - append: + tag: append_email_to_address_55646ca6 field: email.to.address value: '{{{mimecast.Recipient}}}' allow_duplicates: false if: ctx.mimecast?.Recipient != null - rename: + tag: rename_mimecast_SenderDomain_to_source_domain_c7ddd99f field: mimecast.SenderDomain target_field: source.domain ignore_missing: true - rename: + tag: rename_mimecast_sha1_to_email_attachments_file_hash_sha1_69b825ff field: mimecast.sha1 target_field: email.attachments.file.hash.sha1 ignore_missing: true - rename: + tag: rename_mimecast_sha256_to_email_attachments_file_hash_sha256_3d63dd5b field: mimecast.sha256 target_field: email.attachments.file.hash.sha256 ignore_missing: true - rename: + tag: rename_mimecast_Size_to_email_attachments_file_size_69fc9ed9 field: mimecast.Size target_field: email.attachments.file.size ignore_missing: true - rename: + tag: rename_mimecast_fileName_to_email_attachments_file_name_93c938f3 field: mimecast.fileName target_field: email.attachments.file.name ignore_missing: true ### SPAM EVENT THREAD LOGS - set: + tag: set_event_kind_74c2f6bf field: event.kind value: alert if: ctx.mimecast?.SourceIP != null && ctx.mimecast.SourceIP != '' - rename: + tag: rename_mimecast_SourceIP_to_source_ip_72e383b9 field: mimecast.SourceIP target_field: source.ip ignore_missing: true ### SIEM Email Protect Logs - set: + tag: set_event_kind_bf6d1d9d field: event.kind value: alert if: ctx.mimecast?.URL != null && ctx.mimecast.URL != '' - rename: + tag: rename_mimecast_URL_to_url_full_ca0a831c field: mimecast.URL target_field: url.full ignore_missing: true ### SIEM Impersonation logs - set: + tag: set_event_kind_c4d3177b field: event.kind value: alert if: ctx.mimecast?.TaggedMalicious == true || ctx.mimecast?.TaggedMalicious == 'true' - rename: + tag: rename_mimecast_Action_to_event_action_bf2f21ed field: mimecast.Action target_field: event.action ignore_missing: true - rename: + tag: rename_mimecast_Definition_to_rule_name_ed69d869 field: mimecast.Definition target_field: rule.name ignore_missing: true - rename: + tag: rename_mimecast_NewDomain_to_source_domain_b787a52c field: mimecast.NewDomain target_field: source.domain ignore_missing: true ### SIEM TTP Url Logs - rename: + tag: rename_mimecast_reason_to_event_reason_bf831335 field: mimecast.reason target_field: event.reason ignore_missing: true - append: + tag: append_email_to_address_e6a6d0a6 field: email.to.address value: '{{{mimecast.recipient}}}' allow_duplicates: false if: ctx.mimecast?.recipient != null - rename: + tag: rename_mimecast_route_to_email_direction_552b5749 field: mimecast.route target_field: email.direction ignore_missing: true - append: + tag: append_email_from_address_b1b278a6 field: email.from.address value: "{{{mimecast.sender}}}" allow_duplicates: false if: "ctx?.mimecast?.sender != null" - rename: + tag: rename_mimecast_senderDomain_to_source_domain_df51fd3f field: mimecast.senderDomain target_field: source.domain ignore_missing: true - rename: + tag: rename_mimecast_sourceIp_to_source_ip_a8821cb9 field: mimecast.sourceIp target_field: source.ip ignore_missing: true - rename: + tag: rename_mimecast_subject_to_email_subject_3c7206f3 field: mimecast.subject target_field: email.subject ignore_missing: true - rename: + tag: rename_mimecast_url_to_url_full_90397fbc field: mimecast.url target_field: url.full ignore_missing: true - rename: + tag: rename_mimecast_action_to_event_action_75b859cd field: mimecast.action target_field: event.action ignore_missing: true - set: + tag: set_event_created_24b501e1 field: event.created value: "{{{mimecast.datetime}}}" if: 'ctx?.mimecast?.datetime != null' - set: + tag: set_tls_established_253e8c18 field: tls.established value: false if: 'ctx?.tls?.established == "No"' - set: + tag: set_tls_established_dddbc7b3 field: tls.established value: true if: 'ctx?.tls?.established == "Yes"' - rename: + tag: rename_mimecast_Delivered_to_event_outcome_3a0307db field: mimecast.Delivered target_field: event.outcome ignore_missing: true - set: + tag: set_event_outcome_09c2dbc3 field: event.outcome value: "success" if: ctx?.event?.outcome == true - set: + tag: set_event_outcome_fcee2e6f field: event.outcome value: "failure" if: ctx?.event?.outcome == false - set: + tag: set_event_outcome_46e49ef3 field: event.outcome value: "unknown" if: ctx?.event?.outcome == null - lowercase: + tag: lowercase_email_direction_569c0270 field: email.direction ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -461,18 +525,21 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true # Cleanup - remove: + tag: remove_5446d5af description: Cleanup of repeated/unwanted/temporary fields. - field: + field: - mimecast.eventTime - mimecast.Content-Disposition - mimecast.datetime @@ -494,3 +561,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/v2_pipeline.yml b/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/v2_pipeline.yml index bf497746540..9d38443dd8d 100644 --- a/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/v2_pipeline.yml +++ b/packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/v2_pipeline.yml @@ -2,6 +2,7 @@ description: Pipeline for processing Mimecast siem_logs from v2 API. processors: - date: + tag: date_mimecast_timestamp_2f9b8186 description: Use 'mimecast.timestamp' as the '@timestamp' field: mimecast.timestamp if: ctx.mimecast?.timestamp != null @@ -9,12 +10,14 @@ processors: formats: - UNIX_MS - set: + tag: set_event_created_3b69f632 field: event.created copy_from: '@timestamp' if: ctx['@timestamp'] != null ### NOTE LOG TYPE - script: + tag: script_2d3bdcf4 lang: painless params: 'av': 'avlog' @@ -40,6 +43,7 @@ processors: if: ctx.tags?.contains('preserve_duplicate_custom_fields') == true - fingerprint: + tag: fingerprint_5cae5d9e fields: - mimecast.messageId - mimecast.processingId @@ -54,6 +58,7 @@ processors: ### Common - foreach: + tag: foreach_mimecast_recipients_d346f855 if: ctx.mimecast?.recipients instanceof List field: mimecast.recipients processor: @@ -62,6 +67,7 @@ processors: value: '{{{_ingest._value.value}}}' allow_duplicates: false - append: + tag: append_email_to_address_0cfb7a4a field: email.to.address value: '{{{mimecast.recipients}}}' allow_duplicates: false @@ -69,122 +75,150 @@ processors: ### RECEIPT LOGS - rename: + tag: rename_mimecast_action_to_event_action_75b859cd field: mimecast.action target_field: event.action ignore_missing: true - rename: + tag: rename_mimecast_tlsCipher_to_tls_cipher_8ff1bb23 field: mimecast.tlsCipher target_field: tls.cipher ignore_missing: true - rename: + tag: rename_mimecast_direction_to_email_direction_624c8c4d field: mimecast.direction target_field: email.direction ignore_missing: true - rename: + tag: rename_mimecast_receiptErrors_to_error_message_2c144233 field: mimecast.receiptErrors target_field: error.message ignore_missing: true - rename: + tag: rename_mimecast_senderIp_to_source_ip_e17dfbc5 field: mimecast.senderIp target_field: source.ip ignore_missing: true - rename: + tag: rename_mimecast_messageId_to_email_message_id_a8ab0b64 field: mimecast.messageId target_field: email.message_id ignore_missing: true - append: + tag: append_email_from_address_0cbef6e5 field: email.from.address value: '{{{mimecast.senderHeader}}}' allow_duplicates: false if: ctx.mimecast?.senderHeader != null - rename: + tag: rename_mimecast_rejectionCode_to_error_code_39daeafa field: mimecast.rejectionCode target_field: error.code ignore_missing: true - rename: + tag: rename_mimecast_rejectionInfo_to_event_reason_fae81034 field: mimecast.rejectionInfo target_field: event.reason ignore_missing: true - set: + tag: set_event_kind_dee3962d field: event.kind value: alert if: ctx.mimecast?.rejectionType != null && ctx.mimecast.rejectionType != '' - rename: + tag: rename_mimecast_rejectionType_to_error_type_00719f72 field: mimecast.rejectionType target_field: error.type ignore_missing: true - append: + tag: append_email_from_address_3477ed67 field: email.from.address value: '{{{mimecast.senderEnvelope}}}' allow_duplicates: false if: ctx.mimecast?.senderEnvelope != null - rename: + tag: rename_mimecast_subject_to_email_subject_3c7206f3 field: mimecast.subject target_field: email.subject ignore_missing: true - rename: + tag: rename_mimecast_tlsVer_to_tls_version_c151af6a field: mimecast.tlsVer target_field: tls.version ignore_missing: true ### PROCESS LOGS - rename: + tag: rename_mimecast_totalSizeAttachments_to_email_attachments_file_size_d3e9fcbf field: mimecast.totalSizeAttachments target_field: email.attachments.file.size ignore_missing: true - convert: + tag: convert_email_attachments_file_size_to_email_attachments_file_size_36c6954a field: email.attachments.file.size target_field: email.attachments.file.size type: long ignore_missing: true on_failure: - remove: + tag: remove_email_attachments_file_size_08a06211 field: email.attachments.file.size - rename: + tag: rename_mimecast_attachments_to_email_attachments_file_name_e8d63fdc field: mimecast.attachments target_field: email.attachments.file.name ignore_missing: true - set: + tag: set_event_kind_a51a4ef9 field: event.kind value: alert if: ctx.mimecast?.Hld != null && ctx.mimecast.Hld != '' - rename: + tag: rename_mimecast_holdReason_to_event_reason_4ab378fe field: mimecast.holdReason target_field: event.reason ignore_missing: true ### DELIVERY LOGS - rename: + tag: rename_mimecast_destinationIp_to_destination_ip_d24d88c9 field: mimecast.destinationIp target_field: destination.ip ignore_missing: true - rename: + tag: rename_mimecast_deliveryErrors_to_error_message_9deacf97 field: mimecast.deliveryErrors target_field: error.message ignore_missing: true - rename: + tag: rename_mimecast_tlsUsed_to_tls_established_2da09b42 field: mimecast.tlsUsed target_field: tls.established ignore_missing: true - set: + tag: set_tls_established_ee2a9e9b field: tls.established value: true if: ctx.tls?.established instanceof String && ctx.tls.established.equalsIgnoreCase('yes') - set: + tag: set_tls_established_acef8b40 field: tls.established value: false if: ctx.tls?.established instanceof String && ctx.tls.established.equalsIgnoreCase('no') ### AV LOGS - set: + tag: set_event_kind_d195129d field: event.kind value: alert if: ctx.mimecast?.fileExtension != null && ctx.mimecast.fileExtension != '' - rename: + tag: rename_mimecast_fileExtension_to_email_attachments_file_extension_41a48113 field: mimecast.fileExtension target_field: email.attachments.file.extension ignore_missing: true - rename: + tag: rename_mimecast_md5_to_email_attachments_file_hash_md5_639a4925 field: mimecast.md5 target_field: email.attachments.file.hash.md5 ignore_missing: true @@ -195,57 +229,70 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_mimecast_senderDomainInternal_24f7b9bd field: mimecast.senderDomainInternal - append: + tag: append_error_message_113297ee field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_mimecast_sha1_to_email_attachments_file_hash_sha1_69b825ff field: mimecast.sha1 target_field: email.attachments.file.hash.sha1 ignore_missing: true - rename: + tag: rename_mimecast_sha256_to_email_attachments_file_hash_sha256_3d63dd5b field: mimecast.sha256 target_field: email.attachments.file.hash.sha256 ignore_missing: true - rename: + tag: rename_mimecast_fileName_to_email_attachments_file_name_93c938f3 field: mimecast.fileName target_field: email.attachments.file.name ignore_missing: true ### SPAM EVENT THREAD LOGS - set: + tag: set_event_kind_265b6393 field: event.kind value: alert if: ctx.mimecast?.senderIp != null && ctx.mimecast.senderIp != '' - rename: + tag: rename_mimecast_senderIp_to_source_ip_e17dfbc5 field: mimecast.senderIp target_field: source.ip ignore_missing: true ### SIEM Email Protect Logs - set: + tag: set_event_kind_0f82209d field: event.kind value: alert if: ctx.mimecast?.url != null && ctx.mimecast.url != '' - rename: + tag: rename_mimecast_url_to_url_full_90397fbc field: mimecast.url target_field: url.full ignore_missing: true ### SIEM Impersonation logs - set: + tag: set_event_kind_770897fb field: event.kind value: alert if: ctx.mimecast?.taggedMalicious == true || ctx.mimecast?.taggedMalicious == 'true' - rename: + tag: rename_mimecast_action_to_event_action_75b859cd field: mimecast.action target_field: event.action ignore_missing: true - rename: + tag: rename_mimecast_policyDefinition_to_rule_name_ca38c3b3 field: mimecast.policyDefinition target_field: rule.name ignore_missing: true - rename: + tag: rename_mimecast_newDomain_to_source_domain_61e7b24c field: mimecast.newDomain target_field: source.domain ignore_missing: true @@ -256,8 +303,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_mimecast_taggedExternal_770fc1ce field: mimecast.taggedExternal - append: + tag: append_error_message_7e9097aa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -267,70 +316,87 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_mimecast_taggedMalicious_1189f190 field: mimecast.taggedMalicious - append: + tag: append_error_message_4a758765 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' ### SIEM TTP Url Logs - rename: + tag: rename_mimecast_blockReason_to_event_reason_49d8adfa field: mimecast.blockReason target_field: event.reason ignore_missing: true - rename: + tag: rename_mimecast_route_to_email_direction_8174b78e field: mimecast.route target_field: email.direction ignore_missing: true if: ctx.email?.direction == null - append: + tag: append_email_from_address_fe6e5273 field: email.from.address value: "{{{mimecast.sender}}}" allow_duplicates: false if: ctx.mimecast?.sender != null - rename: + tag: rename_mimecast_senderDomain_to_source_domain_df51fd3f field: mimecast.senderDomain target_field: source.domain ignore_missing: true - rename: + tag: rename_mimecast_sourceIp_to_source_ip_a8821cb9 field: mimecast.sourceIp target_field: source.ip ignore_missing: true - rename: + tag: rename_mimecast_subject_to_email_subject_3c7206f3 field: mimecast.subject target_field: email.subject ignore_missing: true - rename: + tag: rename_mimecast_url_to_url_full_90397fbc field: mimecast.url target_field: url.full ignore_missing: true - rename: + tag: rename_mimecast_action_to_event_action_75b859cd field: mimecast.action target_field: event.action ignore_missing: true - rename: + tag: rename_mimecast_Delivered_to_event_outcome_3a0307db field: mimecast.Delivered target_field: event.outcome ignore_missing: true - set: + tag: set_event_outcome_7b299714 field: event.outcome value: "success" if: ctx.event?.outcome == true - set: + tag: set_event_outcome_48eb961a field: event.outcome value: "failure" if: ctx.event?.outcome == false - set: + tag: set_event_outcome_6f947b7c field: event.outcome value: "unknown" if: ctx.event?.outcome == null - lowercase: + tag: lowercase_email_direction_569c0270 field: email.direction ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -339,18 +405,22 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -359,10 +429,12 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true @@ -424,11 +496,13 @@ processors: # Cleanup - rename: + tag: rename_original_to_mimecast_5b6791aa field: original target_field: mimecast override: true ignore_missing: true - remove: + tag: remove_0a5321eb description: Cleanup of repeated/unwanted/temporary fields. field: - mimecast._offset @@ -444,3 +518,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/mimecast/data_stream/threat_intel_malware_customer/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/threat_intel_malware_customer/elasticsearch/ingest_pipeline/default.yml index db21be4fb9e..f1b3e3e38c7 100644 --- a/packages/mimecast/data_stream/threat_intel_malware_customer/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/threat_intel_malware_customer/elasticsearch/ingest_pipeline/default.yml @@ -5,6 +5,7 @@ processors: # Event ECS fields # #################### - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - remove: @@ -19,12 +20,15 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - set: + tag: set_event_kind_a131107f field: event.kind value: enrichment - set: + tag: set_event_category_ab27d5b9 field: event.category - value: [threat,email,malware] + value: [threat, email, malware] - set: + tag: set_event_type_97bb78b9 field: event.type value: [indicator] ###################### @@ -44,11 +48,14 @@ processors: description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null - json: + tag: json_event_original_to_mimecast_7b94ac41 field: event.original target_field: mimecast - drop: + tag: drop_406985dc if: ctx.mimecast?.type != "indicator" || (ctx.mimecast?.objects != null && ctx.mimecast.objects.isEmpty()) - fingerprint: + tag: fingerprint_b0c83cf3 fields: - mimecast.id target_field: "_id" @@ -59,6 +66,7 @@ processors: ##################### ## File indicator operations - date: + tag: date_mimecast_created_f3ab7d03 field: mimecast.created formats: - "yyyy-MM-dd'T'HH:mmz" @@ -73,6 +81,7 @@ processors: - "yyyy-MM-dd'T'HH:mm:ss.SSSZ" if: "ctx.mimecast?.created != null" - date: + tag: date_mimecast_modified_to_threat_indicator_modified_at_04e284eb field: mimecast.modified target_field: threat.indicator.modified_at formats: @@ -88,6 +97,7 @@ processors: - "yyyy-MM-dd'T'HH:mm:ss.SSSZ" if: "ctx.mimecast?.modified != null" - date: + tag: date_mimecast_valid_from_to_threat_indicator_first_seen_e84e46e4 field: mimecast.valid_from target_field: threat.indicator.first_seen formats: @@ -102,58 +112,70 @@ processors: - "yyyy-MM-dd'T'HH:mm:ss.SSSz" - "yyyy-MM-dd'T'HH:mm:ss.SSSZ" if: "ctx.mimecast?.valid_from != null" - - dissect: + - dissect: + tag: dissect_mimecast_pattern_853fddf0 field: mimecast.pattern pattern: "[%{_tmp.threattype}:%{mimecast.pattern_value}.'%{mimecast.pattern_hash_type}' = '%{_tmp.threatvalue}']" if: 'ctx.mimecast?.pattern != null' - - set: + - set: + tag: set_mimecast_value_122340b1 field: mimecast.value value: "{{{_tmp.threatvalue}}}" if: 'ctx?._tmp?.threatvalue != null' - - set: + - set: + tag: set_mimecast_hashtype_302deae2 field: mimecast.hashtype value: "{{{mimecast.pattern_hash_type}}}" if: 'ctx?.mimecast?.pattern_hash_type != null' - rename: + tag: rename__tmp_threattype_to_threat_indicator_type_57aa6470 field: _tmp.threattype target_field: threat.indicator.type ignore_missing: true ignore_failure: true - rename: + tag: rename_mimecast_value_to_threat_indicator_file_hash_sha256_28ff121b field: mimecast.value target_field: threat.indicator.file.hash.sha256 ignore_missing: true if: 'ctx?.mimecast?.hashtype == "SHA-256"' - rename: + tag: rename_mimecast_value_to_threat_indicator_file_hash_sha1_234256b3 field: mimecast.value target_field: threat.indicator.file.hash.sha1 ignore_missing: true if: 'ctx?.mimecast?.hashtype == "SHA-1"' - rename: + tag: rename_mimecast_value_to_threat_indicator_file_hash_md5_90bea4a9 field: mimecast.value target_field: threat.indicator.file.hash.md5 ignore_missing: true if: 'ctx?.mimecast?.hashtype == "MD-5"' - append: + tag: append_related_hash_350e866b field: related.hash value: "{{{threat.indicator.file.hash.sha256}}}" allow_duplicates: false if: 'ctx?.mimecast?.hashtype == "SHA-256"' - append: + tag: append_related_hash_2e34317b field: related.hash value: "{{{threat.indicator.file.hash.sha1}}}" allow_duplicates: false if: 'ctx?.mimecast?.hashtype == "SHA-1"' - append: + tag: append_related_hash_8fcb9967 field: related.hash value: "{{{threat.indicator.file.hash.md5}}}" allow_duplicates: false if: 'ctx?.mimecast?.hashtype == "MD-5"' - set: + tag: set_threat_indicator_type_ac7917ab field: threat.indicator.type value: unknown if: ctx.threat?.indicator?.type == null - foreach: + tag: foreach_mimecast_labels_0a3ddad6 field: mimecast.labels ignore_missing: true processor: @@ -162,35 +184,42 @@ processors: value: "{{{_ingest._value}}}" allow_duplicates: false - grok: + tag: grok_mimecast_description_427abf85 field: mimecast.description patterns: - "^%{GREEDYDATA}Source: %{GREEDYDATA:threat.indicator.provider}" ignore_missing: true ignore_failure: true - dissect: + tag: dissect_mimecast_Content-Disposition_b148edcd field: mimecast.Content-Disposition pattern: "%{?drop->}=\"%{mimecast.logtype_part.1}_%{mimecast.logtype_part.2}_%{?drop->}" ignore_missing: true - set: + tag: set_mimecast_log_type_d7aed698 field: mimecast.log_type value: "{{{mimecast.logtype_part.1}}}_{{{mimecast.logtype_part.2}}}" if: 'ctx?.mimecast?.logtype_part?.1 != null && ctx?.mimecast?.logtype_part?.2 != null' - rename: + tag: rename_mimecast_name_to_email_attachments_file_name_ff5ac5d8 field: mimecast.name target_field: email.attachments.file.name - ignore_missing: true + ignore_missing: true if: 'ctx?.mimecast?.name != null' - split: + tag: split_email_attachments_file_name_to_file_parts_9e1a7769 field: email.attachments.file.name separator: "\\." target_field: file.parts if: 'ctx?.email?.attachments?.file?.name != null' - script: + tag: script_6ac6dfdd lang: painless source: | ctx.email.attachments.file.name = ctx.file.parts[0]; if: 'ctx?.file?.parts !=null && ctx?.file?.parts.length > 1' - script: + tag: script_e4f15236 lang: painless source: | ctx.email.attachments.file.extension = ctx.file.parts[ctx.file.parts.length-1]; @@ -225,6 +254,7 @@ processors: } handleMap(ctx); - remove: + tag: remove_4a2abec2 field: - mimecast.created - message diff --git a/packages/mimecast/data_stream/threat_intel_malware_grid/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/threat_intel_malware_grid/elasticsearch/ingest_pipeline/default.yml index 7dced0d8445..70b16a4d369 100644 --- a/packages/mimecast/data_stream/threat_intel_malware_grid/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/threat_intel_malware_grid/elasticsearch/ingest_pipeline/default.yml @@ -5,6 +5,7 @@ processors: # Event ECS fields # #################### - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - remove: @@ -19,12 +20,15 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - set: + tag: set_event_kind_a131107f field: event.kind value: enrichment - set: + tag: set_event_category_ab27d5b9 field: event.category - value: [threat,email,malware] + value: [threat, email, malware] - set: + tag: set_event_type_97bb78b9 field: event.type value: [indicator] ###################### @@ -44,11 +48,14 @@ processors: description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null - json: + tag: json_event_original_to_mimecast_7b94ac41 field: event.original target_field: mimecast - drop: + tag: drop_406985dc if: ctx.mimecast?.type != "indicator" || (ctx.mimecast?.objects != null && ctx.mimecast.objects.isEmpty()) - fingerprint: + tag: fingerprint_b0c83cf3 fields: - mimecast.id target_field: "_id" @@ -58,6 +65,7 @@ processors: ##################### ## File indicator operations - date: + tag: date_mimecast_created_f3ab7d03 field: mimecast.created formats: - "yyyy-MM-dd'T'HH:mmz" @@ -72,6 +80,7 @@ processors: - "yyyy-MM-dd'T'HH:mm:ss.SSSZ" if: "ctx.mimecast?.created != null" - date: + tag: date_mimecast_modified_to_threat_indicator_modified_at_04e284eb field: mimecast.modified target_field: threat.indicator.modified_at formats: @@ -87,6 +96,7 @@ processors: - "yyyy-MM-dd'T'HH:mm:ss.SSSZ" if: "ctx.mimecast?.modified != null" - date: + tag: date_mimecast_valid_from_to_threat_indicator_first_seen_e84e46e4 field: mimecast.valid_from target_field: threat.indicator.first_seen formats: @@ -102,57 +112,69 @@ processors: - "yyyy-MM-dd'T'HH:mm:ss.SSSZ" if: "ctx.mimecast?.valid_from != null" - dissect: + tag: dissect_mimecast_pattern_c3d728da field: mimecast.pattern pattern: "[%{_tmp.threattype}:%{mimecast.pattern_value}.'%{mimecast.pattern_hash_type}' = '%{_tmp.threatvalue}']" if: 'ctx?.mimecast?.pattern != null && ctx?.mimecast?.type == "indicator"' - set: + tag: set_mimecast_value_b828ebf8 field: mimecast.value value: "{{{_tmp.threatvalue}}}" if: 'ctx?.mimecast?.pattern != null && ctx?.mimecast?.type == "indicator"' - set: + tag: set_mimecast_hashtype_1761fe35 field: mimecast.hashtype value: "{{{mimecast.pattern_hash_type}}}" if: 'ctx?.mimecast?.pattern != null && ctx?.mimecast?.type == "indicator"' - rename: + tag: rename__tmp_threattype_to_threat_indicator_type_272feebb field: _tmp.threattype target_field: threat.indicator.type ignore_missing: true if: 'ctx?.mimecast?.pattern != null' - rename: + tag: rename_mimecast_value_to_threat_indicator_file_hash_sha256_96c3cc85 field: mimecast.value target_field: threat.indicator.file.hash.sha256 ignore_missing: true if: 'ctx?.mimecast?.hashtype == "SHA-256" && ctx?.mimecast?.pattern != null && ctx?.mimecast?.type == "indicator"' - rename: + tag: rename_mimecast_value_to_threat_indicator_file_hash_sha1_e02fb4b1 field: mimecast.value target_field: threat.indicator.file.hash.sha1 ignore_missing: true if: 'ctx?.mimecast?.hashtype == "SHA-1" && ctx?.mimecast?.pattern != null && ctx?.mimecast?.type == "indicator"' - rename: + tag: rename_mimecast_value_to_threat_indicator_file_hash_md5_4805965b field: mimecast.value target_field: threat.indicator.file.hash.md5 ignore_missing: true if: 'ctx?.mimecast?.hashtype == "MD-5" && ctx?.mimecast?.pattern != null && ctx?.mimecast?.type == "indicator"' - append: + tag: append_related_hash_8ecc6059 field: related.hash value: "{{{threat.indicator.file.hash.sha256}}}" allow_duplicates: false if: 'ctx?.mimecast?.hashtype == "SHA-256" && ctx?.mimecast?.pattern != null && ctx?.mimecast?.type == "indicator"' - append: + tag: append_related_hash_3945e69d field: related.hash value: "{{{threat.indicator.file.hash.sha1}}}" allow_duplicates: false if: 'ctx?.mimecast?.hashtype == "SHA-1" && ctx?.mimecast?.pattern != null && ctx?.mimecast?.type == "indicator"' - append: + tag: append_related_hash_0b2de51d field: related.hash value: "{{{threat.indicator.file.hash.md5}}}" allow_duplicates: false if: 'ctx?.mimecast?.hashtype == "MD-5" && ctx?.mimecast?.pattern != null && ctx?.mimecast?.type == "indicator"' - set: + tag: set_threat_indicator_type_b27d7280 field: threat.indicator.type value: unknown if: 'ctx?.threat?.indicator?.type == null' - foreach: + tag: foreach_mimecast_labels_0a3ddad6 field: mimecast.labels ignore_missing: true processor: @@ -161,34 +183,41 @@ processors: value: "{{{_ingest._value}}}" allow_duplicates: false - grok: + tag: grok_mimecast_description_427abf85 field: mimecast.description patterns: - "^%{GREEDYDATA}Source: %{GREEDYDATA:threat.indicator.provider}" ignore_missing: true ignore_failure: true - dissect: + tag: dissect_mimecast_Content-Disposition_b148edcd field: mimecast.Content-Disposition pattern: "%{?drop->}=\"%{mimecast.logtype_part.1}_%{mimecast.logtype_part.2}_%{?drop->}" ignore_missing: true - set: + tag: set_mimecast_log_type_da0ad1e6 field: mimecast.log_type value: "{{{mimecast.logtype_part.1}}}_{{{mimecast.logtype_part.2}}}" if: 'ctx?.mimecast?.logtype_part != null' - rename: + tag: rename_mimecast_name_to_email_attachments_file_name_1a2949cd field: mimecast.name target_field: email.attachments.file.name ignore_missing: true - split: + tag: split_email_attachments_file_name_to_file_parts_9e1a7769 field: email.attachments.file.name separator: "\\." target_field: file.parts if: 'ctx?.email?.attachments?.file?.name != null' - script: + tag: script_6ac6dfdd lang: painless source: | ctx.email.attachments.file.name = ctx.file.parts[0]; if: 'ctx?.file?.parts !=null && ctx?.file?.parts.length > 1' - script: + tag: script_e4f15236 lang: painless source: | ctx.email.attachments.file.extension = ctx.file.parts[ctx.file.parts.length-1]; @@ -223,6 +252,7 @@ processors: } handleMap(ctx); - remove: + tag: remove_4a2abec2 field: - mimecast.created - message diff --git a/packages/mimecast/data_stream/ttp_ap_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/ttp_ap_logs/elasticsearch/ingest_pipeline/default.yml index bddf2f89e3d..c88003c7ca5 100644 --- a/packages/mimecast/data_stream/ttp_ap_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/ttp_ap_logs/elasticsearch/ingest_pipeline/default.yml @@ -3,6 +3,7 @@ description: Pipeline for processing Mimecast ttp_ap_logs. processors: # Generic event/ecs fields we always want to populate. - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - remove: @@ -30,137 +31,165 @@ processors: description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null - json: + tag: json_event_original_to_mimecast_d72eb09c description: Parse 'message' JSON contents into a 'mimecast' object. field: event.original target_field: mimecast - drop: + tag: drop_6ce7222a if: ctx.mimecast?.date == null || (ctx.mimecast?.data != null && ctx.mimecast.data.isEmpty()) - fingerprint: + tag: fingerprint_27d21f1f fields: - mimecast.messageId - mimecast.date target_field: "_id" ignore_missing: true - set: + tag: set_event_category_4861796c field: event.category value: [email] - set: + tag: set_event_kind_964c251d field: event.kind value: alert if: ctx.mimecast?.result == 'malicious' - date: + tag: date_mimecast_date_88412148 description: Use 'mimecast.date' as the '@timestamp' field: mimecast.date timezone: UTC formats: - yyyy-MM-dd'T'HH:mm:ssZ - append: + tag: append_email_from_address_090cee64 field: email.from.address value: "{{{mimecast.senderAddress}}}" if: "ctx?.mimecast?.senderAddress != null" ignore_failure: true - append: + tag: append_email_to_address_9e646f4d field: email.to.address value: "{{{mimecast.recipientAddress}}}" if: "ctx?.mimecast?.recipientAddress != null" ignore_failure: true - dissect: + tag: dissect_email_from_address_969e2d4f field: email.from.address pattern: "<%{email.from.address}>" ignore_missing: true ignore_failure: true - gsub: + tag: gsub_email_from_address_6a1fe514 field: email.from.address pattern: '<>' replacement: '' ignore_missing: true ignore_failure: true - dissect: + tag: dissect_email_to_address_3f03f6e9 field: email.to.address pattern: "<%{email.to.address}>" ignore_missing: true ignore_failure: true - gsub: + tag: gsub_email_to_address_1726742b field: email.to.address pattern: '<>' replacement: '' ignore_missing: true ignore_failure: true - gsub: + tag: gsub_mimecast_actionTriggered_a8cd8df7 field: mimecast.actionTriggered pattern: "," replacement: "" ignore_missing: true - gsub: + tag: gsub_mimecast_actionTriggered_e1aeb826 field: mimecast.actionTriggered pattern: " " replacement: "_" ignore_missing: true - rename: + tag: rename_mimecast_actionTriggered_to_event_action_63fa0a2f field: mimecast.actionTriggered target_field: event.action ignore_missing: true if: 'ctx?.mimecast?.actionTriggered !=null' - rename: + tag: rename_mimecast_subject_to_email_subject_f6b733ff field: mimecast.subject target_field: email.subject ignore_missing: true if: 'ctx?.mimecast?.subject !=null' - rename: + tag: rename_mimecast_messageId_to_email_message_id_eca64d12 field: mimecast.messageId target_field: email.message_id ignore_missing: true if: 'ctx?.mimecast?.messageId !=null' - rename: + tag: rename_mimecast_route_to_email_direction_880aaa48 field: mimecast.route target_field: email.direction ignore_missing: true if: 'ctx?.mimecast?.route !=null' - rename: + tag: rename_mimecast_fileName_to_email_attachments_file_name_23af68fe field: mimecast.fileName target_field: email.attachments.file.name ignore_missing: true if: 'ctx?.mimecast?.fileName !=null' - rename: + tag: rename_mimecast_definition_to_rule_name_7efc08c8 field: mimecast.definition target_field: rule.name ignore_missing: true if: 'ctx?.mimecast?.definition !=null' - rename: + tag: rename_mimecast_fileHash_to_email_attachments_file_hash_sha256_128dae7e field: mimecast.fileHash target_field: email.attachments.file.hash.sha256 ignore_missing: true if: 'ctx.mimecast?.fileHash != null && ctx.mimecast.fileHash.length() == 64' - rename: + tag: rename_mimecast_fileType_to_email_attachments_file_mime_type_f6ee0660 field: mimecast.fileType target_field: email.attachments.file.mime_type ignore_missing: true if: 'ctx?.mimecast?.fileType !=null' - set: + tag: set_event_created_db76d553 field: event.created value: "{{{mimecast.date}}}" if: 'ctx?.mimecast?.date != null' - split: + tag: split_email_attachments_file_name_to_file_parts_9e1a7769 field: email.attachments.file.name separator: "\\." target_field: file.parts if: 'ctx?.email?.attachments?.file?.name != null' - script: + tag: script_e4f15236 lang: painless source: | ctx.email.attachments.file.extension = ctx.file.parts[ctx.file.parts.length-1]; if: 'ctx?.file?.parts !=null && ctx?.file?.parts.length > 1' - append: + tag: append_related_hash_9b6ef41b field: related.hash value: "{{{email.attachments.file.hash.sha256}}}" allow_duplicates: false if: ctx.email?.attachments?.file?.hash?.sha256 != null - lowercase: + tag: lowercase_email_direction_569c0270 field: email.direction ignore_missing: true ### Cleanup - remove: + tag: remove_d5765a6a description: Cleanup of repeated/unwanted/temporary fields. field: - mimecast.date diff --git a/packages/mimecast/data_stream/ttp_ip_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/ttp_ip_logs/elasticsearch/ingest_pipeline/default.yml index 53efbd9ec51..c2ab53f177f 100644 --- a/packages/mimecast/data_stream/ttp_ip_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/ttp_ip_logs/elasticsearch/ingest_pipeline/default.yml @@ -3,6 +3,7 @@ description: Pipeline for processing Mimecast ttp_ip_logs. processors: # Generic event/ecs fields we always want to populate. - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - remove: @@ -30,94 +31,115 @@ processors: description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null - json: + tag: json_event_original_to_mimecast_d72eb09c description: Parse 'message' JSON contents into a 'mimecast' object. field: event.original target_field: mimecast - drop: + tag: drop_dd4ed123 if: ctx.mimecast?.eventTime == null || (ctx.mimecast?.data != null && ctx.mimecast.data.isEmpty()) - fingerprint: + tag: fingerprint_2f1e6f7e fields: - mimecast.messageId - mimecast.eventTime target_field: "_id" ignore_missing: true - date: + tag: date_mimecast_eventTime_a835a222 description: Use 'mimecast.eventTime' as the '@timestamp' field: mimecast.eventTime timezone: UTC formats: - yyyy-MM-dd'T'HH:mm:ssZ - set: + tag: set_event_category_4861796c field: event.category value: [email] - set: + tag: set_event_kind_b058344c field: event.kind value: alert if: ctx.mimecast?.taggedMalicious == true ### - rename: + tag: rename_mimecast_senderIpAddress_to_source_ip_6edea6c5 field: mimecast.senderIpAddress target_field: source.ip ignore_missing: true - append: + tag: append_email_from_address_090cee64 field: email.from.address value: "{{{mimecast.senderAddress}}}" if: "ctx?.mimecast?.senderAddress != null" ignore_failure: true - rename: + tag: rename_mimecast_subject_to_email_subject_3c7206f3 field: mimecast.subject target_field: email.subject ignore_missing: true - rename: + tag: rename_mimecast_action_to_event_action_75b859cd field: mimecast.action target_field: event.action ignore_missing: true - rename: + tag: rename_mimecast_definition_to_rule_name_ce386dc9 field: mimecast.definition target_field: rule.name ignore_missing: true - rename: + tag: rename_mimecast_id_to_event_id_c7737d91 field: mimecast.id target_field: event.id ignore_missing: true - append: + tag: append_email_to_address_9e646f4d field: email.to.address value: "{{{mimecast.recipientAddress}}}" if: "ctx?.mimecast?.recipientAddress != null" ignore_failure: true - rename: + tag: rename_mimecast_messageId_to_email_message_id_a8ab0b64 field: mimecast.messageId target_field: email.message_id ignore_missing: true - rename: + tag: rename_mimecast_similarDomain_to_source_domain_56d4058f field: mimecast.similarDomain target_field: source.domain ignore_missing: true - set: + tag: set_event_created_6dbfe9df field: event.created value: "{{{mimecast.eventTime}}}" if: 'ctx?.mimecast?.eventTime != null' - append: + tag: append_related_ip_549fa858 field: related.ip value: "{{{source.ip}}}" allow_duplicates: false if: 'ctx?.source?.ip != null' - dissect: + tag: dissect_email_from_address_969e2d4f field: email.from.address pattern: "<%{email.from.address}>" ignore_missing: true ignore_failure: true - dissect: + tag: dissect_email_to_address_3f03f6e9 field: email.to.address pattern: "<%{email.to.address}>" ignore_missing: true ignore_failure: true - lowercase: + tag: lowercase_email_direction_569c0270 field: email.direction ignore_missing: true ### Cleanup - remove: + tag: remove_cd4e5bfd description: Cleanup of repeated/unwanted/temporary fields. field: - mimecast.eventTime diff --git a/packages/mimecast/data_stream/ttp_url_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/ttp_url_logs/elasticsearch/ingest_pipeline/default.yml index bf747d3ea2a..081e8a6184f 100644 --- a/packages/mimecast/data_stream/ttp_url_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/ttp_url_logs/elasticsearch/ingest_pipeline/default.yml @@ -3,6 +3,7 @@ description: Pipeline for processing Mimecast ttp_url_logs. processors: # Generic event/ecs fields we always want to populate. - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - remove: @@ -30,115 +31,140 @@ processors: description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null - json: + tag: json_event_original_to_mimecast_d72eb09c description: Parse 'message' JSON contents into a 'mimecast' object. field: event.original target_field: mimecast - drop: + tag: drop_6ce7222a if: ctx.mimecast?.date == null || (ctx.mimecast?.data != null && ctx.mimecast.data.isEmpty()) - fingerprint: + tag: fingerprint_27d21f1f fields: - mimecast.messageId - mimecast.date target_field: "_id" ignore_missing: true - date: + tag: date_mimecast_date_88412148 description: Use 'mimecast.date' as the '@timestamp' field: mimecast.date timezone: UTC formats: - yyyy-MM-dd'T'HH:mm:ssZ - set: + tag: set_event_category_4861796c field: event.category value: [email] - set: + tag: set_event_kind_ad202b20 field: event.kind value: alert if: ctx.mimecast?.scanResult == 'malicious' ### - rename: + tag: rename_mimecast_url_to_url_original_7cf8a0a2 field: mimecast.url target_field: url.original ignore_missing: true - uri_parts: + tag: uri_parts_url_original_a660a8ab field: url.original keep_original: true ignore_missing: true ignore_failure: true - rename: + tag: rename_mimecast_userAwarenessAction_to_event_action_802a8079 field: mimecast.userAwarenessAction target_field: event.action ignore_missing: true - rename: + tag: rename_mimecast_route_to_email_direction_552b5749 field: mimecast.route target_field: email.direction ignore_missing: true - rename: + tag: rename_mimecast_ttpDefinition_to_rule_name_d854d979 field: mimecast.ttpDefinition target_field: rule.name ignore_missing: true - rename: + tag: rename_mimecast_subject_to_email_subject_3c7206f3 field: mimecast.subject target_field: email.subject ignore_missing: true - rename: + tag: rename_mimecast_messageId_to_email_message_id_a8ab0b64 field: mimecast.messageId target_field: email.message_id ignore_missing: true - set: + tag: set_email_from_address_bda9e358 field: email.from.address value: ["{{{mimecast.fromUserEmailAddress}}}"] if: "ctx.mimecast?.fromUserEmailAddress != null" - append: + tag: append_related_user_5753eea6 field: related.user value: "{{{mimecast.fromUserEmailAddress}}}" allow_duplicates: false if: 'ctx.mimecast?.fromUserEmailAddress != null' ignore_failure: true - set: + tag: set_email_to_address_55c4e91f field: email.to.address value: ["{{{mimecast.userEmailAddress}}}"] if: "ctx.mimecast?.userEmailAddress != null" - set: + tag: set_user_email_1e711c32 field: user.email value: ["{{{mimecast.userEmailAddress}}}"] if: "ctx?.mimecast?.userEmailAddress != null" - append: + tag: append_related_user_cdbe6caa field: related.user value: "{{{mimecast.userEmailAddress}}}" allow_duplicates: false if: 'ctx.mimecast?.userEmailAddress != null' ignore_failure: true - convert: + tag: convert_mimecast_sendingIp_to_source_ip_471402f5 field: mimecast.sendingIp target_field: source.ip type: ip ignore_missing: true ignore_failure: true - remove: + tag: remove_mimecast_sendingIp_d48483a4 field: mimecast.sendingIp if: ctx.source?.ip != null ignore_missing: true - set: + tag: set_event_created_db76d553 field: event.created value: "{{{mimecast.date}}}" if: 'ctx?.mimecast?.date != null' - append: + tag: append_related_ip_242edff8 field: related.ip value: "{{{source.ip}}}" allow_duplicates: false if: 'ctx?.source?.ip !=null' - dissect: + tag: dissect_email_from_address_969e2d4f field: email.from.address pattern: "<%{email.from.address}>" ignore_missing: true ignore_failure: true - dissect: + tag: dissect_email_to_address_3f03f6e9 field: email.to.address pattern: "<%{email.to.address}>" ignore_missing: true ignore_failure: true - foreach: + tag: foreach_mimecast_tagMap_DangerousFileExt_8d291c4d field: mimecast.tagMap.DangerousFileExt ignore_missing: true ignore_failure: true @@ -150,6 +176,7 @@ processors: ignore_missing: true ### Cleanup - remove: + tag: remove_44c4fea2 description: Cleanup of repeated/unwanted/temporary fields. field: - mimecast.date diff --git a/packages/mimecast/manifest.yml b/packages/mimecast/manifest.yml index 679502f973c..d2a99bb05fa 100644 --- a/packages/mimecast/manifest.yml +++ b/packages/mimecast/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.3.2" name: mimecast title: "Mimecast" -version: "3.5.1" +version: "3.5.2" description: Collect logs from Mimecast with Elastic Agent. type: integration categories: ["security", "email_security"] diff --git a/packages/miniflux/changelog.yml b/packages/miniflux/changelog.yml index 2fcec188ab6..e864d96e77e 100644 --- a/packages/miniflux/changelog.yml +++ b/packages/miniflux/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.2.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.2.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/miniflux/data_stream/feed_entry/elasticsearch/ingest_pipeline/default.yml b/packages/miniflux/data_stream/feed_entry/elasticsearch/ingest_pipeline/default.yml index a4bec51d4b5..29985933e60 100644 --- a/packages/miniflux/data_stream/feed_entry/elasticsearch/ingest_pipeline/default.yml +++ b/packages/miniflux/data_stream/feed_entry/elasticsearch/ingest_pipeline/default.yml @@ -5,7 +5,7 @@ processors: field: ecs.version tag: set_ecs_version value: 8.17.0 - + - terminate: tag: data_collection_error if: ctx.error?.message != null && ctx.message == null && ctx.event?.original == null @@ -41,6 +41,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -141,7 +142,7 @@ processors: field: json.feed.hide_globally tag: rename_feed_hide_globally target_field: miniflux.feed.hide_globally - - rename: + - rename: field: json.feed.icon.feed_id tag: rename_feed_icon_feed_id target_field: miniflux.feed.icon.feed_id @@ -253,13 +254,14 @@ processors: target_field: miniflux.feed.webhook_url - rename: field: json.feed_id - tag: rename_feed_id + tag: rename_json_feed_id_to_miniflux_feed_id_a1e93bde target_field: miniflux.feed_id - rename: field: json.hash tag: rename_hash target_field: miniflux.hash - append: + tag: append_related_hash_808cebe9 field: related.hash value: "{{{miniflux.hash}}}" - rename: @@ -297,14 +299,17 @@ processors: tag: rename_title target_field: miniflux.title - uri_parts: + tag: uri_parts_json_url_514cc9d1 field: json.url if: ctx.json?.url != null && ctx.json.url != '' on_failure: - set: + tag: set_url_original_f46e88d0 field: url.original copy_from: url.original if: ctx.json?.url != null && ctx.json.url != '' - set: + tag: set_url_full_adb622a8 field: url.full copy_from: url.original if: ctx.url?.original != null && ctx.url.original != '' @@ -320,7 +325,7 @@ processors: # Cleanup - remove: tag: remove_json - field: + field: - json - script: tag: script_to_drop_null_values @@ -346,6 +351,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false @@ -364,4 +370,4 @@ on_failure: - append: field: tags value: preserve_original_event - allow_duplicates: false \ No newline at end of file + allow_duplicates: false diff --git a/packages/miniflux/manifest.yml b/packages/miniflux/manifest.yml index 8a8b65f59f4..68abe512d3a 100644 --- a/packages/miniflux/manifest.yml +++ b/packages/miniflux/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.5 name: miniflux title: "Miniflux RSS reader" -version: 1.2.1 +version: 1.2.2 source: license: "Elastic-2.0" description: Collect RSS feed content from the Miniflux API with Elastic Agent. diff --git a/packages/netskope/changelog.yml b/packages/netskope/changelog.yml index 4281de9aba1..b520df4f3c0 100644 --- a/packages/netskope/changelog.yml +++ b/packages/netskope/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.1.5" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "3.1.4" changes: - description: Add content_type and override_content_type options to the transaction data stream for the azure-blob-storage input. diff --git a/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml b/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml index 832603b7daa..c33706a0ec2 100644 --- a/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml +++ b/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml @@ -12,6 +12,7 @@ processors: add_to_root_conflict_strategy: replace on_failure: - append: + tag: append_error_message_e5154119 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -34,9 +35,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_source_geo_location_lat_2ceab886 field: source.geo.location.lat ignore_missing: true - append: + tag: append_error_message_b4ad3654 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -49,9 +52,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_source_geo_location_lon_2a70ef56 field: source.geo.location.lon ignore_missing: true - append: + tag: append_error_message_9953f288 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -70,9 +75,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_destination_geo_location_lat_204268e4 field: destination.geo.location.lat ignore_missing: true - append: + tag: append_error_message_c864af81 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -85,9 +92,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_destination_geo_location_lon_9a53596c field: destination.geo.location.lon ignore_missing: true - append: + tag: append_error_message_847e9e7d field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -106,9 +115,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_user_geo_location_lat_ff7385ec field: netskope.alerts.user.geo.location.lat ignore_missing: true - append: + tag: append_error_message_3d2d0f78 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -121,9 +132,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_user_geo_location_lon_8c1c8eac field: netskope.alerts.user.geo.location.lon ignore_missing: true - append: + tag: append_error_message_cdecaeb4 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -144,9 +157,11 @@ processors: - UNIX on_failure: - set: + tag: set_timestamp_18055d85 field: '@timestamp' value: "{{{_ingest.timestamp}}}" - append: + tag: append_error_message_b4f64810 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -160,9 +175,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_destination_ip_00d24f79 field: destination.ip ignore_missing: true - append: + tag: append_error_message_1f6112a2 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -176,9 +193,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_user_ip_aa565553 field: netskope.alerts.user.ip ignore_missing: true - append: + tag: append_error_message_d4ab6a93 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -192,9 +211,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_source_ip_fd76d527 field: source.ip ignore_missing: true - append: + tag: append_error_message_6d53d405 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -272,6 +293,7 @@ processors: remove_if_successful: false on_failure: - rename: + tag: rename_netskope_alerts_url_to_netskope_alerts_url_original_90035e18 field: netskope.alerts.url target_field: netskope.alerts.url.original ignore_missing: true @@ -283,6 +305,7 @@ processors: remove_if_successful: false on_failure: - rename: + tag: rename_netskope_alerts_web_url_to_netskope_alerts_web_url_original_4d3108b5 field: netskope.alerts.web.url target_field: netskope.alerts.web.url.original ignore_missing: true @@ -298,6 +321,7 @@ processors: remove_if_successful: false on_failure: - rename: + tag: rename_netskope_alerts_page_url_to_netskope_alerts_page_url_original_3f848ae0 field: netskope.alerts.page.url target_field: netskope.alerts.page.url.original ignore_missing: true @@ -309,6 +333,7 @@ processors: remove_if_successful: false on_failure: - rename: + tag: rename_netskope_alerts_login_url_to_netskope_alerts_login_url_original_8782b50c field: netskope.alerts.login.url target_field: netskope.alerts.login.url.original ignore_missing: true @@ -320,6 +345,7 @@ processors: remove_if_successful: false on_failure: - rename: + tag: rename_netskope_alerts_referer_to_netskope_alerts_referer_original_ce2f609c field: netskope.alerts.referer target_field: netskope.alerts.referer.original ignore_missing: true @@ -342,9 +368,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_managed_app_df0071a1 field: netskope.alerts.managed.app ignore_missing: true - append: + tag: append_error_message_0dde89fb field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -369,9 +397,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_is_alert_15678b42 field: netskope.alerts.is_alert ignore_missing: true - append: + tag: append_error_message_43b2b137 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -401,9 +431,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_is_malicious_834a8cbe field: netskope.alerts.is_malicious ignore_missing: true - append: + tag: append_error_message_d8ef486f field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -428,9 +460,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_aggregated_user_2305ce4b field: netskope.alerts.aggregated.user ignore_missing: true - append: + tag: append_error_message_cda236d7 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -455,9 +489,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_bypass_traffic_ebde337f field: netskope.alerts.bypass.traffic ignore_missing: true - append: + tag: append_error_message_02757338 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -482,9 +518,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_is_user_generated_d85155b8 field: netskope.alerts.is_user_generated ignore_missing: true - append: + tag: append_error_message_f157228a field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -509,9 +547,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_dlp_is_unique_count_7e682f3d field: netskope.alerts.dlp.is_unique_count ignore_missing: true - append: + tag: append_error_message_8305adb3 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -536,9 +576,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_is_file_passwd_protected_bb90b00e field: netskope.alerts.is_file_passwd_protected ignore_missing: true - append: + tag: append_error_message_f0c0f035 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -568,9 +610,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_is_web_universal_connector_4d6ced06 field: netskope.alerts.is_web_universal_connector ignore_missing: true - append: + tag: append_error_message_35181f75 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -595,9 +639,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_acked_cf3f4084 field: netskope.alerts.acked ignore_missing: true - append: + tag: append_error_message_0a2b138e field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -622,9 +668,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_obfuscate_d5804928 field: netskope.alerts.obfuscate ignore_missing: true - append: + tag: append_error_message_7f853f18 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -654,9 +702,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_ml_detection_beb28582 field: netskope.alerts.ml_detection ignore_missing: true - append: + tag: append_error_message_fa62d447 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -681,9 +731,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_shared_is_shared_c8c877db field: netskope.alerts.shared.is_shared ignore_missing: true - append: + tag: append_error_message_53b31674 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -700,9 +752,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_agg_window_ba610c49 field: netskope.alerts.agg.window ignore_missing: true - append: + tag: append_error_message_5b324af2 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -715,9 +769,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_alert_score_4728dd31 field: netskope.alerts.alert.score ignore_missing: true - append: + tag: append_error_message_f362b0a5 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -730,9 +786,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_alert_window_92c1f0ad field: netskope.alerts.alert.window ignore_missing: true - append: + tag: append_error_message_dba484c5 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -745,9 +803,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_asn_474af042 field: netskope.alerts.asn ignore_missing: true - append: + tag: append_error_message_fbe171da field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -760,9 +820,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_bin_timestamp_0256601d field: netskope.alerts.bin.timestamp ignore_missing: true - append: + tag: append_error_message_8db8e49c field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -775,9 +837,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_breach_score_e950273b field: netskope.alerts.breach.score ignore_missing: true - append: + tag: append_error_message_cb728848 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -790,9 +854,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_connection_duration_602ad467 field: netskope.alerts.connection.duration ignore_missing: true - append: + tag: append_error_message_468c721f field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -805,9 +871,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_connection_endtime_414a91e7 field: netskope.alerts.connection.endtime ignore_missing: true - append: + tag: append_error_message_8bfe4881 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -820,9 +888,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_connection_starttime_40de448f field: netskope.alerts.connection.starttime ignore_missing: true - append: + tag: append_error_message_d964d2da field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -835,9 +905,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_count_83333036 field: netskope.alerts.count ignore_missing: true - append: + tag: append_error_message_2646bb8f field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -850,9 +922,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_data_version_95ac9e33 field: netskope.alerts.data.version ignore_missing: true - append: + tag: append_error_message_bea16175 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -865,9 +939,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_destination_geoip_src_62b7e549 field: netskope.alerts.destination.geoip_src ignore_missing: true - append: + tag: append_error_message_4ee672d8 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -880,9 +956,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_dlp_fingerprint_score_2faddc86 field: netskope.alerts.dlp.fingerprint.score ignore_missing: true - append: + tag: append_error_message_9b44ede6 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -895,9 +973,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_dlp_fv_ef60d7a1 field: netskope.alerts.dlp.fv ignore_missing: true - append: + tag: append_error_message_18459573 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -910,9 +990,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_dlp_rule_count_e7828b2e field: netskope.alerts.dlp.rule.count ignore_missing: true - append: + tag: append_error_message_eda64001 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -925,9 +1007,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_dlp_rule_score_36cc4216 field: netskope.alerts.dlp.rule.score ignore_missing: true - append: + tag: append_error_message_e369bafe field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -940,9 +1024,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_dlp_unique_count_af59a119 field: netskope.alerts.dlp.unique_count ignore_missing: true - append: + tag: append_error_message_c5962b16 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -955,9 +1041,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_doc_count_29408bc5 field: netskope.alerts.doc.count ignore_missing: true - append: + tag: append_error_message_036cd110 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -970,9 +1058,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_external_collaborator_count_b670d048 field: netskope.alerts.external.collaborator.count ignore_missing: true - append: + tag: append_error_message_d477e760 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -985,9 +1075,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_external_email_73817f53 field: netskope.alerts.external.email ignore_missing: true - append: + tag: append_error_message_b1b328dc field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1000,9 +1092,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_file_size_b7152563 field: netskope.alerts.file.size ignore_missing: true - append: + tag: append_error_message_a315498c field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1015,9 +1109,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_http_transaction_count_594ad9bc field: netskope.alerts.http_transaction_count ignore_missing: true - append: + tag: append_error_message_bfaa3aed field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1030,9 +1126,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_insertion_epoch_timestamp_35015ace field: netskope.alerts.insertion_epoch_timestamp ignore_missing: true - append: + tag: append_error_message_6855d638 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1045,9 +1143,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_internal_collaborator_count_91915724 field: netskope.alerts.internal.collaborator.count ignore_missing: true - append: + tag: append_error_message_c5d2bbfe field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1060,9 +1160,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_last_modified_timestamp_ded04cb7 field: netskope.alerts.last.modified_timestamp ignore_missing: true - append: + tag: append_error_message_899ea0bb field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1075,9 +1177,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_last_timestamp_1632cf1f field: netskope.alerts.last.timestamp ignore_missing: true - append: + tag: append_error_message_bb70b491 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1090,9 +1194,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_latency_max_526d9035 field: netskope.alerts.latency.max ignore_missing: true - append: + tag: append_error_message_36d776b9 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1105,9 +1211,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_latency_min_ee071811 field: netskope.alerts.latency.min ignore_missing: true - append: + tag: append_error_message_883eff07 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1120,9 +1228,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_latency_total_38cd6da1 field: netskope.alerts.latency.total ignore_missing: true - append: + tag: append_error_message_b2d7ed0d field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1135,9 +1245,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_malsite_active_2382a8b5 field: netskope.alerts.malsite.active ignore_missing: true - append: + tag: append_error_message_1f9c5ee8 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1150,9 +1262,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_malsite_confidence_f5aaca1d field: netskope.alerts.malsite.confidence ignore_missing: true - append: + tag: append_error_message_816438f6 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1165,9 +1279,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_malsite_consecutive_be607f19 field: netskope.alerts.malsite.consecutive ignore_missing: true - append: + tag: append_error_message_e81a0c26 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1180,9 +1296,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_malsite_first_seen_72612865 field: netskope.alerts.malsite.first_seen ignore_missing: true - append: + tag: append_error_message_a2f0a3d6 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1195,9 +1313,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_malsite_hostility_d6f3ed45 field: netskope.alerts.malsite.hostility ignore_missing: true - append: + tag: append_error_message_34c742f5 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1210,9 +1330,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_malsite_last_seen_201f2078 field: netskope.alerts.malsite.last.seen ignore_missing: true - append: + tag: append_error_message_c211ef48 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1225,9 +1347,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_modified_timestamp_56c2f573 field: netskope.alerts.modified.timestamp ignore_missing: true - append: + tag: append_error_message_2dadbce0 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1240,9 +1364,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_modified_date_c918b9a3 field: netskope.alerts.modified.date ignore_missing: true - append: + tag: append_error_message_ff64434a field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1255,9 +1381,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_nonzero_entries_e3e16c11 field: netskope.alerts.nonzero.entries ignore_missing: true - append: + tag: append_error_message_0a69637c field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1270,9 +1398,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_numbytes_f94cb13c field: netskope.alerts.numbytes ignore_missing: true - append: + tag: append_error_message_f28659ed field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1285,9 +1415,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_object_count_de4dd8cb field: netskope.alerts.object.count ignore_missing: true - append: + tag: append_error_message_76c15e4d field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1300,9 +1432,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_processing_time_2b3fe29b field: netskope.alerts.processing.time ignore_missing: true - append: + tag: append_error_message_0e61d5eb field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1315,9 +1449,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_request_cnt_ac706419 field: netskope.alerts.request.cnt ignore_missing: true - append: + tag: append_error_message_a8349567 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1330,9 +1466,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_response_cnt_599dd187 field: netskope.alerts.response.cnt ignore_missing: true - append: + tag: append_error_message_8e010f11 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1345,9 +1483,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_response_content_length_88cdb60a field: netskope.alerts.response.content.length ignore_missing: true - append: + tag: append_error_message_b81d12d6 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1360,9 +1500,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_scan_time_681d6c37 field: netskope.alerts.scan.time ignore_missing: true - append: + tag: append_error_message_d891faf9 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1375,9 +1517,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_server_bytes_470f8427 field: netskope.alerts.server.bytes ignore_missing: true - append: + tag: append_error_message_edab1655 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1390,9 +1534,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_severity_level_id_38ddf60b field: netskope.alerts.severity.level_id ignore_missing: true - append: + tag: append_error_message_e7cb2e24 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1405,9 +1551,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_source_geoip_src_214968f3 field: netskope.alerts.source.geoip_src ignore_missing: true - append: + tag: append_error_message_70fbf61d field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1420,9 +1568,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_start_time_61cf5390 field: netskope.alerts.start_time ignore_missing: true - append: + tag: append_error_message_ebe83a50 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1435,9 +1585,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_statistics_e0ec2198 field: netskope.alerts.statistics ignore_missing: true - append: + tag: append_error_message_665cef41 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1450,9 +1602,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_suppression_end_time_1712c89c field: netskope.alerts.suppression.end.time ignore_missing: true - append: + tag: append_error_message_59836115 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1465,9 +1619,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_suppression_start_time_da6430f4 field: netskope.alerts.suppression.start.time ignore_missing: true - append: + tag: append_error_message_7c716a54 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1480,9 +1636,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_threshold_time_46a77131 field: netskope.alerts.threshold.time ignore_missing: true - append: + tag: append_error_message_8858bb53 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1495,9 +1653,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_threshold_value_e337f541 field: netskope.alerts.threshold.value ignore_missing: true - append: + tag: append_error_message_657c6abb field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1510,9 +1670,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_total_collaborator_count_cfe208ba field: netskope.alerts.total.collaborator.count ignore_missing: true - append: + tag: append_error_message_5aa1ab75 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1525,9 +1687,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_tss_version_c3fbd281 field: netskope.alerts.tss.version ignore_missing: true - append: + tag: append_error_message_9cde0797 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1540,9 +1704,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_updated_46e77032 field: netskope.alerts.updated ignore_missing: true - append: + tag: append_error_message_82afa293 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1555,9 +1721,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_anomaly_magnitude_e4591f64 field: netskope.alerts.anomaly.magnitude ignore_missing: true - append: + tag: append_error_message_e0842c77 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1573,9 +1741,11 @@ processors: ignore_failure: true on_failure: - remove: + tag: remove_netskope_alerts_breach_date_8a22074f field: netskope.alerts.breach.date ignore_missing: true - append: + tag: append_error_message_85bed746 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1588,9 +1758,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_malsite_latitude_05c5f08c field: netskope.alerts.malsite.latitude ignore_missing: true - append: + tag: append_error_message_3bc03c97 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1603,9 +1775,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_malsite_longitude_b531d478 field: netskope.alerts.malsite.longitude ignore_missing: true - append: + tag: append_error_message_82892370 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1618,9 +1792,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_malsite_reputation_d342690c field: netskope.alerts.malsite.reputation ignore_missing: true - append: + tag: append_error_message_9f458e8a field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1633,9 +1809,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_nonzero_percentage_3ac997c8 field: netskope.alerts.nonzero.percentage ignore_missing: true - append: + tag: append_error_message_240c282b field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1648,9 +1826,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alerts_value_3ecaaec7 field: netskope.alerts.value ignore_missing: true - append: + tag: append_error_message_37efeb28 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1663,9 +1843,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_client_bytes_3560a015 field: client.bytes ignore_missing: true - append: + tag: append_error_message_fc875cb3 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1678,9 +1860,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_client_port_d0f4bd69 field: client.port ignore_missing: true - append: + tag: append_error_message_9be71537 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1693,9 +1877,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_destination_port_e4afbc97 field: destination.port ignore_missing: true - append: + tag: append_error_message_066d2314 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1708,9 +1894,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_file_size_421a91d5 field: file.size ignore_missing: true - append: + tag: append_error_message_6ffb123a field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1723,9 +1911,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_source_port_9f1b9acd field: source.port ignore_missing: true - append: + tag: append_error_message_b93b6099 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' @@ -1779,6 +1969,7 @@ processors: } on_failure: - append: + tag: append_error_message_02818d35 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' diff --git a/packages/netskope/data_stream/alerts_events_v2/elasticsearch/ingest_pipeline/default.yml b/packages/netskope/data_stream/alerts_events_v2/elasticsearch/ingest_pipeline/default.yml index 8184b789222..9b74dbd71ea 100644 --- a/packages/netskope/data_stream/alerts_events_v2/elasticsearch/ingest_pipeline/default.yml +++ b/packages/netskope/data_stream/alerts_events_v2/elasticsearch/ingest_pipeline/default.yml @@ -20,6 +20,7 @@ processors: target_field: netskope.alerts_events_v2 on_failure: - append: + tag: append_error_message_0bd2a084 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - pipeline: @@ -33,6 +34,7 @@ processors: tag: pipeline_events ignore_missing_pipeline: true - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/netskope/data_stream/alerts_events_v2/elasticsearch/ingest_pipeline/pipeline_alerts_v2.yml b/packages/netskope/data_stream/alerts_events_v2/elasticsearch/ingest_pipeline/pipeline_alerts_v2.yml index 2ba2e5a4613..147159cf484 100644 --- a/packages/netskope/data_stream/alerts_events_v2/elasticsearch/ingest_pipeline/pipeline_alerts_v2.yml +++ b/packages/netskope/data_stream/alerts_events_v2/elasticsearch/ingest_pipeline/pipeline_alerts_v2.yml @@ -43,6 +43,7 @@ processors: } handleMap(ctx); - json: + tag: json_netskope_alert_v2_custom_attr_to_netskope_alert_v2_custom_attr_a3dbba24 field: netskope.alert_v2.custom_attr target_field: netskope.alert_v2.custom_attr ignore_failure: true @@ -59,8 +60,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_acked_bf324f30 field: netskope.alert_v2.acked - append: + tag: append_error_message_90335acc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -70,8 +73,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_dlp_is_unique_count_41ee188a field: netskope.alert_v2.dlp_is_unique_count - append: + tag: append_error_message_5ec8f3e5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -91,6 +96,7 @@ processors: if: ctx.event?.action != null && ctx.event.action != '' on_failure: - append: + tag: append_error_message_c3d192ea field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - join: @@ -100,6 +106,7 @@ processors: if: ctx.event?.action != null && ctx.event.action != '' on_failure: - append: + tag: append_error_message_372e475e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -123,8 +130,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_total_packets_a04ee6ac field: netskope.alert_v2.total_packets - append: + tag: append_error_message_c1567850 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -148,8 +157,10 @@ processors: if: ctx.netskope?.alert_v2?.breach_date != null && ctx.netskope.alert_v2.breach_date != '' on_failure: - remove: + tag: remove_netskope_alert_v2_breach_date_b237f436 field: netskope.alert_v2.breach_date - append: + tag: append_error_message_9fc01c58 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -159,8 +170,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_breach_score_3d61e1a4 field: netskope.alert_v2.breach_score - append: + tag: append_error_message_388334ae field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -175,8 +188,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_cci_ff5ef3fe field: netskope.alert_v2.cci - append: + tag: append_error_message_d4e06339 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -186,8 +201,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_client_bytes_c1214f9a field: netskope.alert_v2.client_bytes - append: + tag: append_error_message_c4206cdb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -202,8 +219,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_client_packets_9359960e field: netskope.alert_v2.client_packets - append: + tag: append_error_message_5c12c0c5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -218,8 +237,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_conn_duration_30db3d6a field: netskope.alert_v2.conn_duration - append: + tag: append_error_message_1ff45e6b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -233,8 +254,10 @@ processors: if: ctx.netskope?.alert_v2?.conn_endtime != null && ctx.netskope.alert_v2.conn_endtime != '' on_failure: - remove: + tag: remove_netskope_alert_v2_conn_endtime_b4224b4d field: netskope.alert_v2.conn_endtime - append: + tag: append_error_message_e6b1b4fa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -248,8 +271,10 @@ processors: if: ctx.netskope?.alert_v2?.conn_starttime != null && ctx.netskope.alert_v2.conn_starttime != '' on_failure: - remove: + tag: remove_netskope_alert_v2_conn_starttime_84351480 field: netskope.alert_v2.conn_starttime - append: + tag: append_error_message_6342e256 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -274,8 +299,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_dlp_rule_count_d369dca2 field: netskope.alert_v2.dlp_rule_count - append: + tag: append_error_message_00286873 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -285,8 +312,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_dlp_unique_count_191a51e0 field: netskope.alert_v2.dlp_unique_count - append: + tag: append_error_message_b4699244 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -308,8 +337,10 @@ processors: if: ctx.json?.domain_ip != '' on_failure: - remove: + tag: remove_netskope_alert_v2_domain_ip_8cd88283 field: netskope.alert_v2.domain_ip - append: + tag: append_error_message_c3a548d3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -330,8 +361,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_dst_geoip_src_7418363e field: netskope.alert_v2.dst_geoip_src - append: + tag: append_error_message_f9bf72cd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -365,6 +398,7 @@ processors: copy_from: netskope.alert_v2.dst_longitude ignore_empty_value: true - remove: + tag: remove_destination_geo_location_d08b068a field: destination.geo.location if: >- !(ctx.destination?.geo?.location?.lat instanceof double) @@ -413,8 +447,10 @@ processors: if: ctx.json?.dstip != '' on_failure: - remove: + tag: remove_netskope_alert_v2_dstip_153176e5 field: netskope.alert_v2.dstip - append: + tag: append_error_message_72f2d137 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -435,8 +471,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_dstport_af3f11dc field: netskope.alert_v2.dstport - append: + tag: append_error_message_7b21a288 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -456,8 +494,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_file_cls_encrypted_1c134a7c field: netskope.alert_v2.file_cls_encrypted - append: + tag: append_error_message_9fe65aec field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -467,8 +507,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_executable_signed_544eec7a field: netskope.alert_v2.executable_signed - append: + tag: append_error_message_a3d09699 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -489,8 +531,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_file_size_421d90f0 field: netskope.alert_v2.file_size - append: + tag: append_error_message_c7767e6a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -521,8 +565,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_iaas_remediated_1c2229e2 field: netskope.alert_v2.iaas_remediated - append: + tag: append_error_message_26e6c147 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -532,8 +578,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_iaas_remediated_on_d16c3786 field: netskope.alert_v2.iaas_remediated_on - append: + tag: append_error_message_5c2d8c7d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -565,7 +613,7 @@ processors: ignore_empty_value: true - append: field: related.hash - tag: append_alert_v2_local_sha1_into_related_hash + tag: append_related_hash_ee6de02d value: '{{{netskope.alert_v2.local_sha256}}}' allow_duplicates: false if: ctx.netskope?.alert_v2?.local_sha256 != null @@ -592,8 +640,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_numbytes_5e68435a field: netskope.alert_v2.numbytes - append: + tag: append_error_message_cd15995b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -603,8 +653,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_dlp_fingerprint_score_2b618058 field: netskope.alert_v2.dlp_fingerprint_score - append: + tag: append_error_message_cb187a6e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -614,8 +666,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_dlp_rule_score_b7e20a94 field: netskope.alert_v2.dlp_rule_score - append: + tag: append_error_message_8d6eb8cc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -650,8 +704,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_req_cnt_5c7735d4 field: netskope.alert_v2.req_cnt - append: + tag: append_error_message_618c0734 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -666,8 +722,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_resp_cnt_68a2a64c field: netskope.alert_v2.resp_cnt - append: + tag: append_error_message_c3c1b7bc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -677,8 +735,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_server_bytes_38602eda field: netskope.alert_v2.server_bytes - append: + tag: append_error_message_1825bc37 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -693,8 +753,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_server_packets_73d4031e field: netskope.alert_v2.server_packets - append: + tag: append_error_message_c8e0d7a1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -709,8 +771,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_session_duration_b9e2d4c6 field: netskope.alert_v2.session_duration - append: + tag: append_error_message_b6343a49 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -720,8 +784,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_event_v2_alert_v2_30d2106a field: netskope.event_v2.alert_v2 - append: + tag: append_error_message_05cbeb87 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -743,6 +809,7 @@ processors: } on_failure: - append: + tag: append_error_message_002707cb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -755,8 +822,10 @@ processors: if: ctx.netskope?.alert_v2?.end_time != null && ctx.netskope.alert_v2.end_time != '' on_failure: - remove: + tag: remove_netskope_alert_v2_end_time_f1f275cb field: netskope.alert_v2.end_time - append: + tag: append_error_message_a2623166 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -774,8 +843,10 @@ processors: if: ctx.netskope?.alert_v2?.start_time != null && ctx.netskope.alert_v2.start_time != '' on_failure: - remove: + tag: remove_netskope_alert_v2_start_time_78b6ded8 field: netskope.alert_v2.start_time - append: + tag: append_error_message_e2596b5a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -795,8 +866,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_src_geoip_src_4a471e04 field: netskope.alert_v2.src_geoip_src - append: + tag: append_error_message_de928428 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -830,6 +903,7 @@ processors: copy_from: netskope.alert_v2.src_longitude ignore_empty_value: true - remove: + tag: remove_source_geo_location_ca1d347e field: source.geo.location if: ctx.source?.geo?.location?.lat == null || ctx.source.geo.location?.lon == null || ctx.source.geo.location.lat < -90.0 || ctx.source.geo.location.lat > 90.0 || ctx.source.geo.location.lon < -180.0 || ctx.source.geo.location.lon > 180.0 ignore_failure: true @@ -861,8 +935,10 @@ processors: if: ctx.json?.srcip != '' on_failure: - remove: + tag: remove_netskope_alert_v2_srcip_ea539bae field: netskope.alert_v2.srcip - append: + tag: append_error_message_390552a1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -887,8 +963,10 @@ processors: if: ctx.netskope?.alert_v2?.timestamp != null && ctx.netskope.alert_v2.timestamp != '' on_failure: - remove: + tag: remove_netskope_alert_v2_timestamp_4b61105c field: netskope.alert_v2.timestamp - append: + tag: append_error_message_2f3a9ab4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -902,8 +980,10 @@ processors: if: ctx.netskope?.alert_v2?.modified_date != null && ctx.netskope.alert_v2.modified_date != '' on_failure: - remove: + tag: remove_netskope_alert_v2_modified_date_6e092560 field: netskope.alert_v2.modified_date - append: + tag: append_error_message_fd816798 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -951,8 +1031,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_alert_v2_user_confidence_index_333a492e field: netskope.alert_v2.user_confidence_index - append: + tag: append_error_message_906f434f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -972,6 +1054,7 @@ processors: tag: 'user_agent_processor' on_failure: - append: + tag: append_error_message_9004f57e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag fail-{{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -993,8 +1076,10 @@ processors: if: ctx.json?.userip != '' on_failure: - remove: + tag: remove_netskope_alert_v2_userip_401d0ebb field: netskope.alert_v2.userip - append: + tag: append_error_message_0eb7c443 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -1100,7 +1185,7 @@ processors: # Cleanup - script: description: This script processor iterates over the whole document to remove fields with null values. - tag: script_to_drop_null_values + tag: script_dbd65e1a lang: painless source: | void handleMap(Map map) { @@ -1129,6 +1214,7 @@ processors: tag: set_event_kind_to_alert value: alert - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/netskope/data_stream/alerts_events_v2/elasticsearch/ingest_pipeline/pipeline_events_v2.yml b/packages/netskope/data_stream/alerts_events_v2/elasticsearch/ingest_pipeline/pipeline_events_v2.yml index 1f50bd7907b..d4b963a52c0 100644 --- a/packages/netskope/data_stream/alerts_events_v2/elasticsearch/ingest_pipeline/pipeline_events_v2.yml +++ b/packages/netskope/data_stream/alerts_events_v2/elasticsearch/ingest_pipeline/pipeline_events_v2.yml @@ -74,6 +74,7 @@ processors: if: ctx.event?.action != null && ctx.event?.action != '' on_failure: - append: + tag: append_error_message_e97af38b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - join: @@ -83,6 +84,7 @@ processors: if: ctx.event?.action != null && ctx.event?.action != '' on_failure: - append: + tag: append_error_message_3b12f157 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -117,7 +119,7 @@ processors: if: ctx.netskope?.events_v2?.activity != null && ctx.netskope.events_v2.activity.toLowerCase().contains('fail') - set: field: event.outcome - tag: set_event_outcome_to_failure + tag: set_event_outcome_9f52d8f7 value: unknown if: ctx.event?.outcome == null - set: @@ -136,8 +138,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_app_session_id_0318ca71 field: netskope.events_v2.app_session_id - append: + tag: append_error_message_2e9e5494 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -147,8 +151,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_browser_session_id_d4cbd11d field: netskope.events_v2.browser_session_id - append: + tag: append_error_message_9014488d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -158,8 +164,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_cci_c4f8ea03 field: netskope.events_v2.cci - append: + tag: append_error_message_2abd9ac3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -169,8 +177,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_client_bytes_6443f3f9 field: netskope.events_v2.client_bytes - append: + tag: append_error_message_ac71e8db field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -190,8 +200,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_client_packets_1e929595 field: netskope.events_v2.client_packets - append: + tag: append_error_message_1fa442d9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -211,8 +223,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_conn_duration_482874c3 field: netskope.events_v2.conn_duration - append: + tag: append_error_message_9a818f81 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -224,8 +238,10 @@ processors: if: ctx.netskope?.events_v2?.conn_endtime != null && ctx.netskope.events_v2.conn_endtime != '' on_failure: - remove: + tag: remove_netskope_events_v2_conn_endtime_7374b6cb field: netskope.events_v2.conn_endtime - append: + tag: append_error_message_4ad11be7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -237,8 +253,10 @@ processors: if: ctx.netskope?.events_v2?.conn_starttime != null && ctx.netskope.events_v2.conn_starttime != '' on_failure: - remove: + tag: remove_netskope_events_v2_conn_starttime_344b9ab2 field: netskope.events_v2.conn_starttime - append: + tag: append_error_message_9870072b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -248,8 +266,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_connection_id_6835926f field: netskope.events_v2.connection_id - append: + tag: append_error_message_cd166030 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -259,8 +279,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_dlp_incident_id_fd24c953 field: netskope.events_v2.dlp_incident_id - append: + tag: append_error_message_5fcfde6b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -270,8 +292,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_dlp_parent_id_5b9fc457 field: netskope.events_v2.dlp_parent_id - append: + tag: append_error_message_77d2a241 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -281,8 +305,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_dlp_rule_count_9ef90fa9 field: netskope.events_v2.dlp_rule_count - append: + tag: append_error_message_11af235f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -292,8 +318,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_dlp_is_unique_count_ba2c3d33 field: netskope.events_v2.dlp_is_unique_count - append: + tag: append_error_message_c56722e7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -303,8 +331,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_dlp_unique_count_93375199 field: netskope.events_v2.dlp_unique_count - append: + tag: append_error_message_6a49b14a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -316,8 +346,10 @@ processors: if: ctx.netskope?.events_v2?.domain_ip != '' on_failure: - remove: + tag: remove_netskope_events_v2_domain_ip_c03fb4ff field: netskope.events_v2.domain_ip - append: + tag: append_error_message_e3ad1cec field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -338,8 +370,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_dst_geoip_src_b0e19c37 field: netskope.events_v2.dst_geoip_src - append: + tag: append_error_message_84ff404b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -373,8 +407,9 @@ processors: copy_from: netskope.events_v2.dst_longitude ignore_empty_value: true - remove: + tag: remove_destination_geo_location_d08b068a field: destination.geo.location - if: >- + if: >- !(ctx.destination?.geo?.location?.lat instanceof double) || !(ctx.destination.geo.location.lon instanceof double) || ctx.destination.geo.location.lat < -90.0 @@ -421,8 +456,10 @@ processors: if: ctx.netskope?.events_v2?.dstip != '' on_failure: - remove: + tag: remove_netskope_events_v2_dstip_0223efac field: netskope.events_v2.dstip - append: + tag: append_error_message_8fc76d05 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -443,8 +480,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_dstport_1483ec8b field: netskope.events_v2.dstport - append: + tag: append_error_message_2af60970 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -462,8 +501,10 @@ processors: if: ctx.netskope?.events_v2?.end_time != null && ctx.netskope.events_v2.end_time != '' on_failure: - remove: + tag: remove_netskope_events_v2_end_time_6931c773 field: netskope.events_v2.end_time - append: + tag: append_error_message_c4183c91 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -483,8 +524,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_file_size_1a6a5d03 field: netskope.events_v2.file_size - append: + tag: append_error_message_148bb786 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -516,8 +559,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_incident_id_34e51a8f field: netskope.events_v2.incident_id - append: + tag: append_error_message_80b4b73a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -536,8 +581,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_latest_incident_id_666949b1 field: netskope.events_v2.latest_incident_id - append: + tag: append_error_message_d2502412 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -558,8 +605,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_numbytes_52880fe9 field: netskope.events_v2.numbytes - append: + tag: append_error_message_cfb8a3bb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -601,8 +650,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_pid_b61493e7 field: netskope.events_v2.pid - append: + tag: append_error_message_0fe38bf7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -632,8 +683,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_req_cnt_e1b3801f field: netskope.events_v2.req_cnt - append: + tag: append_error_message_551c22e8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -643,8 +696,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_request_id_98c7b471 field: netskope.events_v2.request_id - append: + tag: append_error_message_699146bf field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -654,8 +709,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_resp_cnt_cf59f4d1 field: netskope.events_v2.resp_cnt - append: + tag: append_error_message_fa00e10e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -665,8 +722,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_response_time_125bb583 field: netskope.events_v2.response_time - append: + tag: append_error_message_c91299cf field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -676,8 +735,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_server_bytes_c5c63bc9 field: netskope.events_v2.server_bytes - append: + tag: append_error_message_c5615037 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -697,8 +758,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_server_packets_4a043b2d field: netskope.events_v2.server_packets - append: + tag: append_error_message_ee79ff35 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -718,11 +781,14 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_session_duration_4a561335 field: netskope.events_v2.session_duration - append: + tag: append_error_message_209421a1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: + tag: script_f54f584a description: Set event severity based on severity. if: ctx.netskope?.events_v2?.severity != null lang: painless @@ -736,6 +802,7 @@ processors: ctx.event.severity = params.get(ctx.netskope.events_v2.severity.toLowerCase()); on_failure: - append: + tag: append_error_message_6e3db03c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -750,8 +817,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_src_geoip_src_8b4d52e7 field: netskope.events_v2.src_geoip_src - append: + tag: append_error_message_e15501cc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -785,6 +854,7 @@ processors: copy_from: netskope.events_v2.src_longitude ignore_empty_value: true - remove: + tag: remove_source_geo_location_a6f3b91b field: source.geo.location if: >- !(ctx.source?.geo?.location?.lat instanceof double) @@ -801,8 +871,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_srcport_a607c6db field: netskope.events_v2.srcport - append: + tag: append_error_message_50d080ef field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -823,8 +895,10 @@ processors: if: ctx.netskope?.events_v2?.srcip != '' on_failure: - remove: + tag: remove_netskope_events_v2_srcip_6ad5e3eb field: netskope.events_v2.srcip - append: + tag: append_error_message_ba1655a1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -851,8 +925,10 @@ processors: if: ctx.netskope?.events_v2?.start_time != null && ctx.netskope.events_v2.start_time != '' on_failure: - remove: + tag: remove_netskope_events_v2_start_time_0f28087c field: netskope.events_v2.start_time - append: + tag: append_error_message_ba11af3b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -869,8 +945,10 @@ processors: if: ctx.netskope?.events_v2?.timestamp != null && ctx.netskope.events_v2.timestamp != '' on_failure: - remove: + tag: remove_netskope_events_v2_timestamp_89af60d2 field: netskope.events_v2.timestamp - append: + tag: append_error_message_f51fcea3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -896,8 +974,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_total_packets_17b54547 field: netskope.events_v2.total_packets - append: + tag: append_error_message_4ddeee8c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -912,8 +992,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_v2_transaction_id_6622f129 field: netskope.events_v2.transaction_id - append: + tag: append_error_message_3695b014 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -940,6 +1022,7 @@ processors: if: ctx.user?.email != null on_failure: - append: + tag: append_error_message_5e1446ed field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -966,6 +1049,7 @@ processors: tag: 'user_agent_processor' on_failure: - append: + tag: append_error_message_9a5cea21 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag fail-{{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -976,8 +1060,10 @@ processors: if: ctx.netskope?.events_v2?.userip != '' on_failure: - remove: + tag: remove_netskope_events_v2_userip_bd92f588 field: netskope.events_v2.userip - append: + tag: append_error_message_cee1e6b1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -1090,7 +1176,7 @@ processors: # Cleanup - script: description: This script processor iterates over the whole document to remove fields with null values. - tag: script_to_drop_null_values + tag: script_dbd65e1a lang: painless source: | void handleMap(Map map) { @@ -1120,6 +1206,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/netskope/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/netskope/data_stream/events/elasticsearch/ingest_pipeline/default.yml index dfe7160197a..d59ad934abe 100644 --- a/packages/netskope/data_stream/events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/netskope/data_stream/events/elasticsearch/ingest_pipeline/default.yml @@ -2,6 +2,7 @@ description: Pipeline for Netskope events. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - json: @@ -11,125 +12,156 @@ processors: add_to_root_conflict_strategy: replace on_failure: - append: + tag: append_error_message_e5154119 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - convert: + tag: convert_source_geo_location_lat_0625cb92 field: source.geo.location.lat type: double ignore_missing: true on_failure: - remove: + tag: remove_source_geo_location_lat_75629d59 field: source.geo.location.lat ignore_missing: true - append: + tag: append_error_message_eaad4fd5 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_source_geo_location_lon_a17ca842 field: source.geo.location.lon type: double ignore_missing: true on_failure: - remove: + tag: remove_source_geo_location_lon_6a22bc15 field: source.geo.location.lon ignore_missing: true - append: + tag: append_error_message_b279d695 field: error.message value: '{{{_ingest.on_failure_message}}}' - remove: + tag: remove_source_geo_location_a9885a63 field: source.geo.location if: ctx.source?.geo?.location?.lat == null || ctx.source?.geo?.location?.lon == null || ctx.source?.geo?.location?.lat < -90.0 || ctx.source?.geo?.location?.lat > 90.0 || ctx.source?.geo?.location?.lon < -180.0 || ctx.source?.geo?.location?.lon > 180.0 ignore_failure: true ignore_missing: true - convert: + tag: convert_destination_geo_location_lat_244b0b1e field: destination.geo.location.lat type: double ignore_missing: true on_failure: - remove: + tag: remove_destination_geo_location_lat_6dc03fbc field: destination.geo.location.lat ignore_missing: true - append: + tag: append_error_message_1383debd field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_destination_geo_location_lon_42965e62 field: destination.geo.location.lon type: double ignore_missing: true on_failure: - remove: + tag: remove_destination_geo_location_lon_7d018348 field: destination.geo.location.lon ignore_missing: true - append: + tag: append_error_message_d8b02391 field: error.message value: '{{{_ingest.on_failure_message}}}' - remove: + tag: remove_destination_geo_location_b32d7fc2 field: destination.geo.location if: ctx.destination?.geo?.location?.lat == null || ctx.destination?.geo?.location?.lon == null || ctx.destination?.geo?.location?.lat < -90.0 || ctx.destination?.geo?.location?.lat > 90.0 || ctx.destination?.geo?.location?.lon < -180.0 || ctx.destination?.geo?.location?.lon > 180.0 ignore_failure: true ignore_missing: true - convert: + tag: convert_netskope_events_slc_geo_location_lat_e0211d96 field: netskope.events.slc.geo.location.lat type: double ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_slc_geo_location_lat_36e8ffee field: netskope.events.slc.geo.location.lat ignore_missing: true - append: + tag: append_error_message_0742f5fd field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_slc_geo_location_lon_fb1fb33a field: netskope.events.slc.geo.location.lon type: double ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_slc_geo_location_lon_3a07588a field: netskope.events.slc.geo.location.lon ignore_missing: true - append: + tag: append_error_message_c640d711 field: error.message value: '{{{_ingest.on_failure_message}}}' - remove: + tag: remove_netskope_events_slc_geo_location_4a8eaa8c field: netskope.events.slc.geo.location if: ctx.netskope?.events?.slc?.geo?.location?.lat == null || ctx.netskope?.events?.slc?.geo?.location?.lon == null || ctx.netskope?.events?.slc?.geo?.location?.lat < -90.0 || ctx.netskope?.events?.slc?.geo?.location?.lat > 90.0 || ctx.netskope?.events?.slc?.geo?.location?.lon < -180.0 || ctx.netskope?.events?.slc?.geo?.location?.lon > 180.0 ignore_failure: true ignore_missing: true - convert: + tag: convert_netskope_events_user_geo_location_lat_ff800c36 field: netskope.events.user.geo.location.lat type: double ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_user_geo_location_lat_d1ba1b19 field: netskope.events.user.geo.location.lat ignore_missing: true - append: + tag: append_error_message_a12c29d5 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_user_geo_location_lon_997a79d6 field: netskope.events.user.geo.location.lon type: double ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_user_geo_location_lon_a05affcd field: netskope.events.user.geo.location.lon ignore_missing: true - append: + tag: append_error_message_1341704d field: error.message value: '{{{_ingest.on_failure_message}}}' - remove: + tag: remove_netskope_events_user_geo_location_ee16056f field: netskope.events.user.geo.location if: ctx.netskope?.events?.user?.geo?.location?.lat == null || ctx.netskope?.events?.user?.geo?.location?.lon == null || ctx.netskope?.events?.user?.geo?.location?.lat < -90.0 || ctx.netskope?.events?.user?.geo?.location?.lat > 90.0 || ctx.netskope?.events?.user?.geo?.location?.lon < -180.0 || ctx.netskope?.events?.user?.geo?.location?.lon > 180.0 ignore_failure: true ignore_missing: true - date: + tag: date_timestamp_6e4706a9 field: '@timestamp' if: ctx['@timestamp'] != null && ctx['@timestamp'] != '' formats: @@ -137,355 +169,437 @@ processors: - UNIX on_failure: - set: + tag: set_timestamp_8651ee11 field: '@timestamp' value: "{{{_ingest.timestamp}}}" - append: + tag: append_error_message_5b2000bd field: error.message value: Unable to parse the value of Timestamp field, therefore setting the value of Timestamp field to current time. - convert: + tag: convert_destination_ip_20bdc587 field: destination.ip if: ctx.destination?.ip != null && ctx.destination.ip != '' type: ip ignore_missing: true on_failure: - remove: + tag: remove_destination_ip_9fff1bb9 field: destination.ip ignore_missing: true - append: + tag: append_error_message_95b73462 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_user_ip_d4c05271 field: netskope.events.user.ip if: ctx.netskope?.events?.user?.ip != null && ctx.netskope.events.user.ip != '' type: ip ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_user_ip_899f4658 field: netskope.events.user.ip ignore_missing: true - append: + tag: append_error_message_0ae49420 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_source_ip_0c94baef field: source.ip if: ctx.source?.ip != null && ctx.source.ip != '' type: ip ignore_missing: true on_failure: - remove: + tag: remove_source_ip_6dd4b89a field: source.ip ignore_missing: true - append: + tag: append_error_message_640132c2 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_related_ip_51dd17d0 field: related.ip value: '{{{destination.ip}}}' if: ctx.destination?.ip != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_ip_167f4f4a field: related.ip value: '{{{netskope.events.user.ip}}}' if: ctx.netskope?.events?.user?.ip != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_ip_30d15214 field: related.ip value: '{{{source.ip}}}' if: ctx.source?.ip != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_hosts_d0d56b04 field: related.hosts value: '{{{destination.domain}}}' if: ctx.destination?.domain != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_hosts_e6462550 field: related.hosts value: '{{{host.hostname}}}' if: ctx.host?.hostname != null allow_duplicates: false ignore_failure: true - user_agent: + tag: user_agent_user_agent_original_6a886ea1 field: user_agent.original ignore_failure: true - geoip: + tag: geoip_source_ip_to_source_geo_49e0d2f9 field: source.ip if: ctx.source?.ip != null && ctx.source.ip != '' target_field: source.geo ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_geo_bfcb4967 field: destination.ip if: ctx.destination?.ip != null && ctx.destination.ip != '' target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_netskope_events_user_ip_to_netskope_events_user_geo_b1679ac7 field: netskope.events.user.ip if: ctx.netskope?.events?.user?.ip != null && ctx.netskope.events.user.ip != '' target_field: netskope.events.user.geo ignore_missing: true - uri_parts: + tag: uri_parts_netskope_events_url_to_netskope_events_url_d46bf4b3 field: netskope.events.url target_field: netskope.events.url keep_original: true remove_if_successful: false on_failure: - rename: + tag: rename_netskope_events_url_to_netskope_events_url_original_20bccfc1 field: netskope.events.url target_field: netskope.events.url.original ignore_missing: true - uri_parts: + tag: uri_parts_netskope_events_web_url_to_netskope_events_web_url_fd4f5877 field: netskope.events.web.url target_field: netskope.events.web.url keep_original: true remove_if_successful: false on_failure: - rename: + tag: rename_netskope_events_web_url_to_netskope_events_web_url_original_3a49a1dd field: netskope.events.web.url target_field: netskope.events.web.url.original ignore_missing: true - uri_parts: + tag: uri_parts_netskope_events_login_url_to_netskope_events_login_url_61f828db field: netskope.events.login.url target_field: netskope.events.login.url keep_original: true remove_if_successful: false on_failure: - rename: + tag: rename_netskope_events_login_url_to_netskope_events_login_url_original_adfe2a9f field: netskope.events.login.url target_field: netskope.events.login.url.original ignore_missing: true - json: + tag: json_netskope_events_site_d0d7e391 field: netskope.events.site ignore_failure: true - json: + tag: json_netskope_events_app_name_e35c830c field: netskope.events.app.name ignore_failure: true - lowercase: + tag: lowercase_network_protocol_d249a697 ignore_failure: true field: network.protocol - uri_parts: + tag: uri_parts_netskope_events_referer_to_netskope_events_referer_c44ab9ef field: netskope.events.referer target_field: netskope.events.referer keep_original: true remove_if_successful: false on_failure: - rename: + tag: rename_netskope_events_referer_to_netskope_events_referer_original_6673af25 field: netskope.events.referer target_field: netskope.events.referer.original ignore_missing: true - set: + tag: set_netskope_events_managed_app_10059bfa field: netskope.events.managed_app value: true if: "['yes', 'true'].contains(ctx.netskope?.events?.managed_app?.toString()?.toLowerCase())" ignore_failure: true - set: + tag: set_netskope_events_managed_app_30252afc field: netskope.events.managed_app value: false if: "['no', 'false'].contains(ctx.netskope?.events?.managed_app?.toString()?.toLowerCase())" ignore_failure: true - convert: + tag: convert_netskope_events_managed_app_afaa81a1 field: netskope.events.managed_app type: boolean ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_managed_app_b3e58501 field: netskope.events.managed_app ignore_missing: true - append: + tag: append_error_message_033ebf08 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_netskope_events_is_bypass_traffic_743cb80a field: netskope.events.is_bypass_traffic value: true if: "['yes', 'true'].contains(ctx.netskope?.events?.is_bypass_traffic?.toString()?.toLowerCase())" ignore_failure: true - set: + tag: set_netskope_events_is_bypass_traffic_917a8510 field: netskope.events.is_bypass_traffic value: false if: "['no', 'false'].contains(ctx.netskope?.events?.is_bypass_traffic?.toString()?.toLowerCase())" ignore_failure: true - convert: + tag: convert_netskope_events_is_bypass_traffic_a389ca79 field: netskope.events.is_bypass_traffic type: boolean ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_is_bypass_traffic_82745077 field: netskope.events.is_bypass_traffic ignore_missing: true - append: + tag: append_error_message_2d5598cc field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_netskope_events_is_unique_count_07e81ac6 field: netskope.events.is_unique_count value: true if: "['yes', 'true'].contains(ctx.netskope?.events?.is_unique_count?.toString()?.toLowerCase())" ignore_failure: true - set: + tag: set_netskope_events_is_unique_count_1c31c7ac field: netskope.events.is_unique_count value: false if: "['no', 'false'].contains(ctx.netskope?.events?.is_unique_count?.toString()?.toLowerCase())" ignore_failure: true - convert: + tag: convert_netskope_events_is_unique_count_80261e17 field: netskope.events.is_unique_count type: boolean ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_is_unique_count_bc0e91fe field: netskope.events.is_unique_count ignore_missing: true - append: + tag: append_error_message_e6a238da field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_netskope_events_user_is_aggregated_aded787f field: netskope.events.user.is_aggregated value: true if: "['yes', 'true'].contains(ctx.netskope?.events?.user?.is_aggregated?.toString()?.toLowerCase())" ignore_failure: true - set: + tag: set_netskope_events_user_is_aggregated_9a726417 field: netskope.events.user.is_aggregated value: false if: "['no', 'false'].contains(ctx.netskope?.events?.user?.is_aggregated?.toString()?.toLowerCase())" ignore_failure: true - convert: + tag: convert_netskope_events_user_is_aggregated_a24a1cc7 field: netskope.events.user.is_aggregated type: boolean ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_user_is_aggregated_c2753d61 field: netskope.events.user.is_aggregated ignore_missing: true - append: + tag: append_error_message_3674afda field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_netskope_events_alert_is_present_83a02513 field: netskope.events.alert.is_present value: true if: "['yes', 'true'].contains(ctx.netskope?.events?.alert?.is_present?.toString()?.toLowerCase())" ignore_failure: true - set: + tag: set_netskope_events_alert_is_present_62329ecb field: netskope.events.alert.is_present value: false if: "['no', 'false'].contains(ctx.netskope?.events?.alert?.is_present?.toString()?.toLowerCase())" ignore_failure: true - convert: + tag: convert_netskope_events_alert_is_present_00eb907f field: netskope.events.alert.is_present type: boolean ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_alert_is_present_005bc676 field: netskope.events.alert.is_present ignore_missing: true - append: + tag: append_error_message_cc7eff2a field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_netskope_events_user_generated_04ef725f field: netskope.events.user.generated value: true if: "['yes', 'true'].contains(ctx.netskope?.events?.user?.generated?.toString()?.toLowerCase())" ignore_failure: true - set: + tag: set_netskope_events_user_generated_76cb473f field: netskope.events.user.generated value: false if: "['no', 'false'].contains(ctx.netskope?.events?.user?.generated?.toString()?.toLowerCase())" ignore_failure: true - convert: + tag: convert_netskope_events_user_generated_5521b8d7 field: netskope.events.user.generated type: boolean ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_user_generated_eb966ab2 field: netskope.events.user.generated ignore_missing: true - append: + tag: append_error_message_85705aea field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_netskope_events_ack_75ce7fde field: netskope.events.ack value: true if: "['yes', 'true'].contains(ctx.netskope?.events?.ack?.toString()?.toLowerCase())" ignore_failure: true - set: + tag: set_netskope_events_ack_75b3b5b4 field: netskope.events.ack value: false if: "['no', 'false'].contains(ctx.netskope?.events?.ack?.toString()?.toLowerCase())" ignore_failure: true - convert: + tag: convert_netskope_events_ack_1b9188f5 field: netskope.events.ack type: boolean ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_ack_5f7f7171 field: netskope.events.ack ignore_missing: true - append: + tag: append_error_message_bb1c2a20 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_netskope_events_is_malicious_0a13dadc field: netskope.events.is_malicious value: true if: "['yes', 'true'].contains(ctx.netskope?.events?.is_malicious?.toString()?.toLowerCase())" ignore_failure: true - set: + tag: set_netskope_events_is_malicious_a916e2b0 field: netskope.events.is_malicious value: false if: "['no', 'false'].contains(ctx.netskope?.events?.is_malicious?.toString()?.toLowerCase())" ignore_failure: true - convert: + tag: convert_netskope_events_is_malicious_0a63d6b7 field: netskope.events.is_malicious type: boolean ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_is_malicious_77954fb5 field: netskope.events.is_malicious ignore_missing: true - append: + tag: append_error_message_89638fea field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_netskope_events_obfuscate_3728a416 field: netskope.events.obfuscate value: true if: "['yes', 'true'].contains(ctx.netskope?.events?.obfuscate?.toString()?.toLowerCase())" ignore_failure: true - set: + tag: set_netskope_events_obfuscate_e161575c field: netskope.events.obfuscate value: false if: "['no', 'false'].contains(ctx.netskope?.events?.obfuscate?.toString()?.toLowerCase())" ignore_failure: true - convert: + tag: convert_netskope_events_obfuscate_fdbca19f field: netskope.events.obfuscate type: boolean ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_obfuscate_99e4a052 field: netskope.events.obfuscate ignore_missing: true - append: + tag: append_error_message_44b06faa field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_netskope_events_shared_is_shared_bb49cdd3 field: netskope.events.shared.is_shared value: true if: "['yes', 'true'].contains(ctx.netskope?.events?.shared?.is_shared?.toString()?.toLowerCase())" ignore_failure: true - set: + tag: set_netskope_events_shared_is_shared_627eb4cb field: netskope.events.shared.is_shared value: false if: "['no', 'false'].contains(ctx.netskope?.events?.shared?.is_shared?.toString()?.toLowerCase())" ignore_failure: true - convert: + tag: convert_netskope_events_shared_is_shared_161378bb field: netskope.events.shared.is_shared type: boolean ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_shared_is_shared_b9371fcb field: netskope.events.shared.is_shared ignore_missing: true - append: + tag: append_error_message_5a7a1976 field: error.message value: '{{{_ingest.on_failure_message}}}' - date: + tag: date_netskope_events_modified_at_to_netskope_events_modified_at_1117922e field: netskope.events.modified_at if: ctx.netskope?.events?.modified_at != null && ctx.netskope.events.modified_at != '' target_field: netskope.events.modified_at @@ -493,567 +607,721 @@ processors: - UNIX on_failure: - remove: + tag: remove_netskope_events_modified_at_c1ca2b8a field: netskope.events.modified_at ignore_missing: true - append: + tag: append_error_message_08750a7b field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_client_bytes_dbc10287 field: netskope.events.client.bytes type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_client_bytes_affa1ea6 field: netskope.events.client.bytes ignore_missing: true - append: + tag: append_error_message_0831cc08 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_client_packets_b4c00adb field: netskope.events.client.packets type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_client_packets_465fda98 field: netskope.events.client.packets ignore_missing: true - append: + tag: append_error_message_487f3cfc field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_connection_duration_1b9655c3 field: netskope.events.connection.duration type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_connection_duration_68bd67bc field: netskope.events.connection.duration ignore_missing: true - append: + tag: append_error_message_8d3201dc field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_connection_end_time_c65820cd field: netskope.events.connection.end_time type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_connection_end_time_2b1e938d field: netskope.events.connection.end_time ignore_missing: true - append: + tag: append_error_message_8e36e6fe field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_connection_start_time_3bdafe03 field: netskope.events.connection.start_time type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_connection_start_time_bc71d5c0 field: netskope.events.connection.start_time ignore_missing: true - append: + tag: append_error_message_eba7203c field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_count_0c131725 field: netskope.events.count type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_count_461a575b field: netskope.events.count ignore_missing: true - append: + tag: append_error_message_d37a467a field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_destination_geoip_source_ec6ae4ef field: netskope.events.destination.geoip.source type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_destination_geoip_source_277cda7d field: netskope.events.destination.geoip.source ignore_missing: true - append: + tag: append_error_message_c3c8f510 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_dlp_count_8786e619 field: netskope.events.dlp.count type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_dlp_count_9fc09b8d field: netskope.events.dlp.count ignore_missing: true - append: + tag: append_error_message_0acf95fa field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_dlp_fingerprint_score_2166833f field: netskope.events.dlp.fingerprint.score type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_dlp_fingerprint_score_281bc4a8 field: netskope.events.dlp.fingerprint.score ignore_missing: true - append: + tag: append_error_message_5674b4fc field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_dlp_fv_f1f0db87 field: netskope.events.dlp.fv type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_dlp_fv_de4b84a6 field: netskope.events.dlp.fv ignore_missing: true - append: + tag: append_error_message_0dfbed00 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_dlp_score_e81bdd6b field: netskope.events.dlp.score type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_dlp_score_cdd165fe field: netskope.events.dlp.score ignore_missing: true - append: + tag: append_error_message_7d22c3ec field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_dlp_unique_count_e534e92b field: netskope.events.dlp.unique_count type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_dlp_unique_count_1bb87a99 field: netskope.events.dlp.unique_count ignore_missing: true - append: + tag: append_error_message_2b8b50dc field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_domain_shared_with_3ab8229b field: netskope.events.domain_shared_with type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_domain_shared_with_09fbc37d field: netskope.events.domain_shared_with ignore_missing: true - append: + tag: append_error_message_2f7b9754 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_external_collaborator_count_001d86fb field: netskope.events.external_collaborator_count type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_external_collaborator_count_141eeb66 field: netskope.events.external_collaborator_count ignore_missing: true - append: + tag: append_error_message_3e1aa830 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_http_transaction_count_dd1417eb field: netskope.events.http_transaction_count type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_http_transaction_count_ae4d6b13 field: netskope.events.http_transaction_count ignore_missing: true - append: + tag: append_error_message_68e750fc field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_insertion_timestamp_a9701d81 field: netskope.events.insertion.timestamp type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_insertion_timestamp_d6801a63 field: netskope.events.insertion.timestamp ignore_missing: true - append: + tag: append_error_message_60ab26b6 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_internal_collaborator_count_e2c454a3 field: netskope.events.internal_collaborator_count type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_internal_collaborator_count_3baf768c field: netskope.events.internal_collaborator_count ignore_missing: true - append: + tag: append_error_message_679e676c field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_last_timestamp_11ee04ff field: netskope.events.last.timestamp type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_last_timestamp_2244d9be field: netskope.events.last.timestamp ignore_missing: true - append: + tag: append_error_message_bdd12740 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_latency_max_86c7676b field: netskope.events.latency.max type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_latency_max_21abf760 field: netskope.events.latency.max ignore_missing: true - append: + tag: append_error_message_af2817bc field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_latency_min_2be437bb field: netskope.events.latency.min type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_latency_min_06159f9a field: netskope.events.latency.min ignore_missing: true - append: + tag: append_error_message_143dd570 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_latency_total_8139a1c7 field: netskope.events.latency.total type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_latency_total_bdf6c79c field: netskope.events.latency.total ignore_missing: true - append: + tag: append_error_message_cd9e2fec field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_metric_value_0369fdb3 field: netskope.events.metric_value type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_metric_value_8d610b5a field: netskope.events.metric_value ignore_missing: true - append: + tag: append_error_message_1302ff24 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_num_sessions_1e610c0b field: netskope.events.num_sessions type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_num_sessions_44a6e740 field: netskope.events.num_sessions ignore_missing: true - append: + tag: append_error_message_7e158cec field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_numbytes_6afbc473 field: netskope.events.numbytes type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_numbytes_0b202cbb field: netskope.events.numbytes ignore_missing: true - append: + tag: append_error_message_b9031ee4 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_object_count_71c5b583 field: netskope.events.object.count type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_object_count_da4db8e2 field: netskope.events.object.count ignore_missing: true - append: + tag: append_error_message_1df4c59c field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_path_id_d53a7519 field: netskope.events.path_id type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_path_id_cce9ac51 field: netskope.events.path_id ignore_missing: true - append: + tag: append_error_message_720c6d2e field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_request_count_58b73d2b field: netskope.events.request.count type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_request_count_80b50c4e field: netskope.events.request.count ignore_missing: true - append: + tag: append_error_message_e32f7ea8 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_response_content_length_2ef73bd3 field: netskope.events.response.content.length type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_response_content_length_85e09e50 field: netskope.events.response.content.length ignore_missing: true - append: + tag: append_error_message_1a369afc field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_response_count_704ecd9f field: netskope.events.response.count type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_response_count_7cb11822 field: netskope.events.response.count ignore_missing: true - append: + tag: append_error_message_26254018 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_run_id_b7c3f857 field: netskope.events.run_id type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_run_id_1d534f33 field: netskope.events.run_id ignore_missing: true - append: + tag: append_error_message_86200930 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_scan_time_ba5b10bb field: netskope.events.scan.time type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_scan_time_2012235e field: netskope.events.scan.time ignore_missing: true - append: + tag: append_error_message_3a10b700 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_server_bytes_d75a44d7 field: netskope.events.server.bytes type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_server_bytes_8df8f9ba field: netskope.events.server.bytes ignore_missing: true - append: + tag: append_error_message_7203f490 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_server_packets_ab16283b field: netskope.events.server.packets type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_server_packets_55fde8bc field: netskope.events.server.packets ignore_missing: true - append: + tag: append_error_message_ed5d8d5c field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_session_packets_9a363e9d field: netskope.events.session.packets type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_session_packets_a386cb5b field: netskope.events.session.packets ignore_missing: true - append: + tag: append_error_message_5d66f8ca field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_session_duration_473f6387 field: netskope.events.session.duration type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_session_duration_963a7c1a field: netskope.events.session.duration ignore_missing: true - append: + tag: append_error_message_8eb24bf0 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_source_geoip_src_034f7fcf field: netskope.events.source.geoip_src type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_source_geoip_src_a29326fe field: netskope.events.source.geoip_src ignore_missing: true - append: + tag: append_error_message_b7e39cc0 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_suppression_end_time_75a388eb field: netskope.events.suppression.end_time type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_suppression_end_time_e58befb0 field: netskope.events.suppression.end_time ignore_missing: true - append: + tag: append_error_message_187f636c field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_suppression_start_time_064ea487 field: netskope.events.suppression.start_time type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_suppression_start_time_e2479677 field: netskope.events.suppression.start_time ignore_missing: true - append: + tag: append_error_message_fc710208 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_threshold_f8ab4945 field: netskope.events.threshold type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_threshold_df2df4a7 field: netskope.events.threshold ignore_missing: true - append: + tag: append_error_message_8dcf6da2 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_total_packets_b6d996db field: netskope.events.total_packets type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_total_packets_9549859e field: netskope.events.total_packets ignore_missing: true - append: + tag: append_error_message_cefe26bc field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_total_collaborator_count_fb04b07b field: netskope.events.total.collaborator_count type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_total_collaborator_count_00075b78 field: netskope.events.total.collaborator_count ignore_missing: true - append: + tag: append_error_message_465a53bc field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_netskope_events_tunnel_up_time_0bfd66d3 field: netskope.events.tunnel.up_time type: long ignore_missing: true on_failure: - remove: + tag: remove_netskope_events_tunnel_up_time_14484757 field: netskope.events.tunnel.up_time ignore_missing: true - append: + tag: append_error_message_9bb68c64 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_client_bytes_ed7a9c03 field: client.bytes type: long ignore_missing: true on_failure: - remove: + tag: remove_client_bytes_eaf55a98 field: client.bytes ignore_missing: true - append: + tag: append_error_message_a4b8bffc field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_client_nat_ip_6236c74a field: client.nat.ip if: ctx.client?.nat?.ip != null && ctx.client.nat.ip != '' type: ip ignore_missing: true on_failure: - remove: + tag: remove_client_nat_ip_f3c24518 field: client.nat.ip ignore_missing: true - append: + tag: append_error_message_bef37539 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_client_packets_ec6e5367 field: client.packets type: long ignore_missing: true on_failure: - remove: + tag: remove_client_packets_5f3bb63e field: client.packets ignore_missing: true - append: + tag: append_error_message_f8307af0 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_destination_port_3b5fe1ff field: destination.port type: long ignore_missing: true on_failure: - remove: + tag: remove_destination_port_7f9d7943 field: destination.port ignore_missing: true - append: + tag: append_error_message_f0268ec8 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_file_size_117b45b1 field: file.size type: long ignore_missing: true on_failure: - remove: + tag: remove_file_size_b57452d7 field: file.size ignore_missing: true - append: + tag: append_error_message_1769782e field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_source_port_b70360cb field: source.port type: long ignore_missing: true on_failure: - remove: + tag: remove_source_port_ffa6dcfa field: source.port ignore_missing: true - append: + tag: append_error_message_8504b140 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_server_bytes_b70bee1b field: server.bytes type: long ignore_missing: true on_failure: - remove: + tag: remove_server_bytes_4eac3edc field: server.bytes ignore_missing: true - append: + tag: append_error_message_1c69123c field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_server_packets_9718528f field: server.packets type: long ignore_missing: true on_failure: - remove: + tag: remove_server_packets_80ee9c92 field: server.packets ignore_missing: true - append: + tag: append_error_message_79bf3ab8 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_user_roles_3139b7d8 field: user.roles value: ['{{{user.roles}}}'] if: ctx.user?.roles instanceof String - script: + tag: script_32cda030 if: ctx.file?.mime_type != null lang: painless source: >- @@ -1100,9 +1368,11 @@ processors: } on_failure: - append: + tag: append_error_message_01da083c field: error.message - value: '{{{_ingest.on_failure_message}}}' + value: '{{{_ingest.on_failure_message}}}' - script: + tag: script_c3eea815 description: Drops null/empty values recursively lang: painless source: | diff --git a/packages/netskope/data_stream/transaction/elasticsearch/ingest_pipeline/default.yml b/packages/netskope/data_stream/transaction/elasticsearch/ingest_pipeline/default.yml index 35cb286e317..32e9ee6e311 100644 --- a/packages/netskope/data_stream/transaction/elasticsearch/ingest_pipeline/default.yml +++ b/packages/netskope/data_stream/transaction/elasticsearch/ingest_pipeline/default.yml @@ -21,6 +21,7 @@ processors: target_field: netskope.transaction on_failure: - append: + tag: append_error_message_cd9f41da field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -61,6 +62,7 @@ processors: } on_failure: - append: + tag: append_error_message_788a3d12 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -112,6 +114,7 @@ processors: if: ctx.event?.action != null && ctx.event.action != '' on_failure: - append: + tag: append_error_message_c3d192ea field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - join: @@ -121,6 +124,7 @@ processors: if: ctx.event?.action != null && ctx.event.action != '' on_failure: - append: + tag: append_error_message_372e475e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -155,8 +159,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_transaction_bytes_45496f5e field: netskope.transaction.bytes - append: + tag: append_error_message_bf2c3a5e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -167,8 +173,10 @@ processors: if: ctx.netskope?.transaction?.c_ip != '' on_failure: - remove: + tag: remove_netskope_transaction_c_ip_09a3cd8c field: netskope.transaction.c_ip - append: + tag: append_error_message_b6281b58 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -190,8 +198,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_transaction_cs_bytes_ee912925 field: netskope.transaction.cs_bytes - append: + tag: append_error_message_12b3bfb8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -211,8 +221,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_transaction_x_c_latitude_18081d47 field: netskope.transaction.x_c_latitude - append: + tag: append_error_message_aa1c3093 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -222,8 +234,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_transaction_x_c_longitude_ae81b70d field: netskope.transaction.x_c_longitude - append: + tag: append_error_message_564fbe34 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -237,6 +251,7 @@ processors: copy_from: netskope.transaction.x_c_longitude ignore_empty_value: true - remove: + tag: remove_client_geo_location_f9d98cb1 field: client.geo.location if: >- !(ctx.client?.geo?.location?.lat instanceof double) @@ -274,8 +289,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_transaction_rs_status_4308cab7 field: netskope.transaction.rs_status - append: + tag: append_error_message_2ecf7a72 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -292,8 +309,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_transaction_sc_status_313ad86c field: netskope.transaction.sc_status - append: + tag: append_error_message_52e6ded6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -325,8 +344,10 @@ processors: if: ctx.netskope?.transaction?.s_ip != '' on_failure: - remove: + tag: remove_netskope_transaction_s_ip_5714205a field: netskope.transaction.s_ip - append: + tag: append_error_message_e78d9956 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -348,8 +369,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_transaction_sc_bytes_1ee0bc45 field: netskope.transaction.sc_bytes - append: + tag: append_error_message_c5672638 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -365,8 +388,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_transaction_x_cs_app_cci_245e18f2 field: netskope.transaction.x_cs_app_cci - append: + tag: append_error_message_7b22ca18 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -378,8 +403,10 @@ processors: if: ctx.netskope?.transaction?.x_cs_dst_ip != '' on_failure: - remove: + tag: remove_netskope_transaction_x_cs_dst_ip_264c1dd0 field: netskope.transaction.x_cs_dst_ip - append: + tag: append_error_message_e6d403b2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -401,8 +428,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_transaction_x_cs_dst_port_0416ebea field: netskope.transaction.x_cs_dst_port - append: + tag: append_error_message_111f9e32 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -427,8 +456,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_transaction_x_s_latitude_0d215447 field: netskope.transaction.x_s_latitude - append: + tag: append_error_message_ab698883 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -438,8 +469,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_transaction_x_s_longitude_b9c9ea4d field: netskope.transaction.x_s_longitude - append: + tag: append_error_message_47216f44 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -453,6 +486,7 @@ processors: copy_from: netskope.transaction.x_s_longitude ignore_empty_value: true - remove: + tag: remove_destination_geo_location_d08b068a field: destination.geo.location if: >- !(ctx.destination?.geo?.location?.lat instanceof double) @@ -490,8 +524,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_transaction_cs_uri_port_4b7892d6 field: netskope.transaction.cs_uri_port - append: + tag: append_error_message_d66e4efe field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -527,8 +563,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_transaction_x_cs_ip_connect_xff_99571a7a field: netskope.transaction.x_cs_ip_connect_xff - append: + tag: append_error_message_917ffcb6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -546,8 +584,10 @@ processors: if: ctx.netskope?.transaction?.x_cs_ip_xff != '' on_failure: - remove: + tag: remove_netskope_transaction_x_cs_ip_xff_513d697c field: netskope.transaction.x_cs_ip_xff - append: + tag: append_error_message_fec026bd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -565,8 +605,10 @@ processors: if: ctx.netskope?.transaction?.x_cs_src_ip != '' on_failure: - remove: + tag: remove_netskope_transaction_x_cs_src_ip_69a78540 field: netskope.transaction.x_cs_src_ip - append: + tag: append_error_message_1a2f9e2d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -589,8 +631,10 @@ processors: if: ctx.netskope?.transaction?.x_cs_src_ip_egress != '' on_failure: - remove: + tag: remove_netskope_transaction_x_cs_src_ip_egress_5554287c field: netskope.transaction.x_cs_src_ip_egress - append: + tag: append_error_message_f40345dd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -607,8 +651,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_transaction_x_cs_src_port_353c64d5 field: netskope.transaction.x_cs_src_port - append: + tag: append_error_message_72f9e4de field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -625,8 +671,10 @@ processors: if: ctx.netskope?.transaction?.x_cs_userip != '' on_failure: - remove: + tag: remove_netskope_transaction_x_cs_userip_fee1ac38 field: netskope.transaction.x_cs_userip - append: + tag: append_error_message_407956dd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -644,8 +692,10 @@ processors: if: ctx.netskope?.transaction?.x_sr_dst_ip != '' on_failure: - remove: + tag: remove_netskope_transaction_x_sr_dst_ip_641115b4 field: netskope.transaction.x_sr_dst_ip - append: + tag: append_error_message_32a3ef53 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -662,8 +712,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_transaction_x_sr_dst_port_60f488ff field: netskope.transaction.x_sr_dst_port - append: + tag: append_error_message_5164a9de field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -675,8 +727,10 @@ processors: if: ctx.netskope?.transaction?.x_sr_src_ip != '' on_failure: - remove: + tag: remove_netskope_transaction_x_sr_src_ip_ecabdcd8 field: netskope.transaction.x_sr_src_ip - append: + tag: append_error_message_25b96f90 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -693,8 +747,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_netskope_transaction_x_sr_src_port_eb6c8eec field: netskope.transaction.x_sr_src_port - append: + tag: append_error_message_ffca8daa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -706,8 +762,10 @@ processors: if: ctx.netskope?.transaction?.x_ssl_policy_dst_ip != '' on_failure: - remove: + tag: remove_netskope_transaction_x_ssl_policy_dst_ip_c07896c4 field: netskope.transaction.x_ssl_policy_dst_ip - append: + tag: append_error_message_fb8d769b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -725,8 +783,10 @@ processors: if: ctx.netskope?.transaction?.x_ssl_policy_src_ip != '' on_failure: - remove: + tag: remove_netskope_transaction_x_ssl_policy_src_ip_74191b44 field: netskope.transaction.x_ssl_policy_src_ip - append: + tag: append_error_message_a30c9f28 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -744,8 +804,10 @@ processors: if: ctx.netskope?.transaction?.date != null && ctx.netskope.transaction.date != '' on_failure: - remove: + tag: remove_netskope_transaction_date_5c40eb05 field: netskope.transaction.date - append: + tag: append_error_message_a0e9eb12 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -757,8 +819,10 @@ processors: if: ctx.netskope?.transaction?.x_c_local_time != null && ctx.netskope.transaction.x_c_local_time != '' on_failure: - remove: + tag: remove_netskope_transaction_x_c_local_time_24182b17 field: netskope.transaction.x_c_local_time - append: + tag: append_error_message_feae57d2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -770,8 +834,10 @@ processors: if: ctx.netskope?.transaction?.x_cs_timestamp != null && ctx.netskope.transaction.x_cs_timestamp != '' on_failure: - remove: + tag: remove_netskope_transaction_x_cs_timestamp_62467d9f field: netskope.transaction.x_cs_timestamp - append: + tag: append_error_message_c2aafbce field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -788,8 +854,10 @@ processors: if: ctx.netskope?.transaction?.x_r_cert_enddate != null && ctx.netskope.transaction.x_r_cert_enddate != '' on_failure: - remove: + tag: remove_netskope_transaction_x_r_cert_enddate_a7ea4110 field: netskope.transaction.x_r_cert_enddate - append: + tag: append_error_message_ee9df6a7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -806,8 +874,10 @@ processors: if: ctx.netskope?.transaction?.x_r_cert_startdate != null && ctx.netskope.transaction.x_r_cert_startdate != '' on_failure: - remove: + tag: remove_netskope_transaction_x_r_cert_startdate_d8612aa3 field: netskope.transaction.x_r_cert_startdate - append: + tag: append_error_message_7885a70f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -816,6 +886,7 @@ processors: copy_from: netskope.transaction.x_r_cert_startdate ignore_empty_value: true - split: + tag: split_netskope_transaction_x_cs_app_tags_f698a278 field: netskope.transaction.x_cs_app_tags separator: "," preserve_trailing: true @@ -823,11 +894,14 @@ processors: if: ctx.netskope?.transaction?.x_cs_app_tags != '' on_failure: - remove: + tag: remove_netskope_transaction_x_cs_app_tags_c67023c8 field: netskope.transaction.x_cs_app_tags - append: + tag: append_error_message_8c9687f2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - split: + tag: split_netskope_transaction_x_ssl_policy_categories_c8e1708c field: netskope.transaction.x_ssl_policy_categories separator: "," preserve_trailing: true @@ -835,11 +909,14 @@ processors: if: ctx.netskope?.transaction?.x_ssl_policy_categories != '' on_failure: - remove: + tag: remove_netskope_transaction_x_ssl_policy_categories_c82cebec field: netskope.transaction.x_ssl_policy_categories - append: + tag: append_error_message_5f811fc2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - split: + tag: split_netskope_transaction_x_sr_headers_name_4b98e578 field: netskope.transaction.x_sr_headers_name separator: "," preserve_trailing: true @@ -847,11 +924,14 @@ processors: if: ctx.netskope?.transaction?.x_sr_headers_name != '' on_failure: - remove: + tag: remove_netskope_transaction_x_sr_headers_name_0e38381c field: netskope.transaction.x_sr_headers_name - append: + tag: append_error_message_dbfa2a3e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - split: + tag: split_netskope_transaction_x_sr_headers_value_4ecb16ca field: netskope.transaction.x_sr_headers_value separator: "," preserve_trailing: true @@ -859,8 +939,10 @@ processors: if: ctx.netskope?.transaction?.x_sr_headers_value != '' on_failure: - remove: + tag: remove_netskope_transaction_x_sr_headers_value_e7f71890 field: netskope.transaction.x_sr_headers_value - append: + tag: append_error_message_a3e0141c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -936,7 +1018,7 @@ processors: - dissect: field: netskope.transaction.x_cs_ssl_version tag: dissect_tls_version_from_transaction_x_cs_ssl_version - pattern: '%{tls.version_protocol}v%{tls.version}' + pattern: '%{tls.version_protocol}v%{tls.version}' ignore_failure: true - lowercase: field: tls.version_protocol @@ -1010,6 +1092,7 @@ processors: tag: 'user_agent_processor' on_failure: - append: + tag: append_error_message_33848231 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag fail-{{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: @@ -1017,7 +1100,7 @@ processors: - log.file.path - log.offset tag: remove_log_fields - if: ctx.aws?.s3?.bucket != null && ctx.aws.s3.object != null + if: ctx.aws?.s3?.bucket != null && ctx.aws.s3.object != null ignore_missing: true - remove: field: @@ -1072,7 +1155,7 @@ processors: # Cleanup - script: description: This script processor iterates over the whole document to remove fields with null values. - tag: script_to_drop_null_values + tag: script_dbd65e1a lang: painless source: | void handleMap(Map map) { @@ -1102,6 +1185,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/netskope/manifest.yml b/packages/netskope/manifest.yml index 410e1478353..141369deba8 100644 --- a/packages/netskope/manifest.yml +++ b/packages/netskope/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: netskope title: "Netskope" -version: "3.1.4" +version: "3.1.5" description: Collect logs from Netskope with Elastic Agent. type: integration categories: diff --git a/packages/okta/changelog.yml b/packages/okta/changelog.yml index 6850a024165..33cdd7f5127 100644 --- a/packages/okta/changelog.yml +++ b/packages/okta/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.15.1" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "3.15.0" changes: - description: Use new `release` field for agentless deployment mode to establish as beta. diff --git a/packages/okta/data_stream/system/elasticsearch/ingest_pipeline/no_use_flattened_debug.yml b/packages/okta/data_stream/system/elasticsearch/ingest_pipeline/no_use_flattened_debug.yml index a8f6bb4751e..9cef9e18c98 100644 --- a/packages/okta/data_stream/system/elasticsearch/ingest_pipeline/no_use_flattened_debug.yml +++ b/packages/okta/data_stream/system/elasticsearch/ingest_pipeline/no_use_flattened_debug.yml @@ -244,4 +244,8 @@ on_failure: {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/okta/data_stream/system/elasticsearch/ingest_pipeline/use_flattened_debug.yml b/packages/okta/data_stream/system/elasticsearch/ingest_pipeline/use_flattened_debug.yml index aef389042df..7c4d1093520 100644 --- a/packages/okta/data_stream/system/elasticsearch/ingest_pipeline/use_flattened_debug.yml +++ b/packages/okta/data_stream/system/elasticsearch/ingest_pipeline/use_flattened_debug.yml @@ -213,4 +213,8 @@ on_failure: {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/okta/manifest.yml b/packages/okta/manifest.yml index 160e146ad91..45cf7bd23f4 100644 --- a/packages/okta/manifest.yml +++ b/packages/okta/manifest.yml @@ -1,6 +1,6 @@ name: okta title: Okta -version: "3.15.0" +version: "3.15.1" description: Collect and parse event logs from Okta API with Elastic Agent. type: integration format_version: "3.6.1" diff --git a/packages/opencanary/changelog.yml b/packages/opencanary/changelog.yml index 8283278078c..4404f01315f 100644 --- a/packages/opencanary/changelog.yml +++ b/packages/opencanary/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.0.1" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.0.0" changes: - description: Release package as GA. diff --git a/packages/opencanary/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/opencanary/data_stream/events/elasticsearch/ingest_pipeline/default.yml index 443d19a3d31..bccab9cf54f 100755 --- a/packages/opencanary/data_stream/events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/opencanary/data_stream/events/elasticsearch/ingest_pipeline/default.yml @@ -54,7 +54,6 @@ processors: - yyyy-MM-dd HH:mm:ss.SSSSSS ignore_failure: true - ################################## ## Determine basic event fields ## ################################## @@ -84,7 +83,7 @@ processors: - set: if: ctx.opencanary?.logtype != null && ctx.opencanary.logtype >= 2000 description: Set event.category as a network/IDS event if it is of an approriate logtype - tag: set_event_category + tag: set_event_category_e22e799a field: event.category value: - network @@ -194,7 +193,6 @@ processors: } else { ctx.log['logger'] = params.get(logType); } - - set: if: ctx.opencanary?.logtype == null description: Set log.logger to LOG_BASE_ERROR as previous script process will have been skipped @@ -439,7 +437,7 @@ processors: # TFTP Events - rename: description: Rename TFTP FILENAME field to opencanary.tftp.filename - tag: rename_opencanary_logdata_FILENAME + tag: rename_opencanary_logdata_FILENAME_to_opencanary_tftp_filename_0b2e7468 if: ctx.log?.logger == 'LOG_TFTP' field: opencanary.logdata.FILENAME target_field: opencanary.tftp.filename @@ -495,8 +493,6 @@ processors: ignore_missing: true ignore_failure: true - - # NTP events - rename: description: Rename NTP command field to opencanary.ntp.cmd @@ -530,7 +526,6 @@ processors: copy_from: '@timestamp' ignore_failure: true - ###################################### ## Parse Destination Details to ECS ## ###################################### @@ -558,6 +553,7 @@ processors: ignore_missing: true on_failure: - set: + tag: set_destination_domain_2a110a87 copy_from: destination.address field: destination.domain override: true @@ -569,7 +565,6 @@ processors: field: destination.domain ignore_empty_value: true - ############################### ## Parse Host Details to ECS ## ############################### @@ -580,7 +575,6 @@ processors: field: host.name ignore_empty_value: true - ################################# ## Parse Source Details to ECS ## ################################# @@ -608,6 +602,7 @@ processors: ignore_missing: true on_failure: - set: + tag: set_source_domain_829b7f23 copy_from: source.address field: source.domain override: true @@ -628,7 +623,7 @@ processors: ignore_failure: true - grok: description: Parse source and destination MAC addresses from opencanary.logdata.MAC if present - tag: gsub_opencanary_logdata_MAC + tag: grok_opencanary_logdata_MAC_95eba733 field: opencanary.logdata.MAC patterns: - "%{MAC:destination.mac}-%{MAC:source.mac}%{GREEDYDATA}" @@ -649,7 +644,6 @@ processors: field: source.domain ignore_empty_value: true - ################################## ## Parse Network Details to ECS ## ################################## @@ -797,12 +791,14 @@ processors: ## GeoIP Enrichment ## ###################### - geoip: + tag: geoip_source_ip_to_source_geo_5b302835 description: "Enrich event with source IP GeoIP data" field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_9e053401 description: "Enrich event with source IP GeoIP ASN contextual data" field: source.ip target_field: source.as @@ -813,23 +809,26 @@ processors: ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - - geoip: + tag: geoip_destination_ip_to_destination_geo_46a31e70 description: "Enrich event with destination IP GeoIP data" field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_08af3990 description: "Enrich event with destination IP GeoIP ASN contextual data" field: destination.ip target_field: destination.as @@ -840,16 +839,17 @@ processors: ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - ############## ## Clean Up ## ############## @@ -925,10 +925,12 @@ processors: dropEmptyFields(ctx); - set: + tag: set_event_kind_92954dfa field: event.kind value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/opencanary/manifest.yml b/packages/opencanary/manifest.yml index 112a8802318..01bf711ce36 100644 --- a/packages/opencanary/manifest.yml +++ b/packages/opencanary/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.1.3 name: opencanary title: "OpenCanary" -version: "1.0.0" +version: "1.0.1" description: "This integration collects and parses logs from OpenCanary honeypots." type: integration categories: diff --git a/packages/panw_cortex_xdr/changelog.yml b/packages/panw_cortex_xdr/changelog.yml index 846056b06e9..b6670655c34 100644 --- a/packages/panw_cortex_xdr/changelog.yml +++ b/packages/panw_cortex_xdr/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.7.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "2.7.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/panw_cortex_xdr/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml b/packages/panw_cortex_xdr/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml index 867530a0f52..4ceb8451ae1 100644 --- a/packages/panw_cortex_xdr/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml +++ b/packages/panw_cortex_xdr/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml @@ -2,6 +2,7 @@ description: Pipeline for Palo Alto XDR API. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - terminate: @@ -20,6 +21,7 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - rename: + tag: rename_message_to_event_original_a6172163 description: Use 'event.original' if present (e.g. when reprocessing), otherwise 'message'. field: message target_field: event.original @@ -32,22 +34,28 @@ processors: description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null - set: + tag: set_event_kind_39295792 field: event.kind value: alert - append: + tag: append_event_category_557b9332 field: event.category value: malware allow_duplicates: false - append: + tag: append_event_type_3f43a39d field: event.type value: info allow_duplicates: false - json: + tag: json_event_original_to_panw_cortex_xdr_8d2b549c field: event.original target_field: panw_cortex.xdr - drop: + tag: drop_c1f77990 if: ctx.panw_cortex?.xdr?.reply?.result_count == 0 - script: + tag: script_7e38081f description: Drops null/empty values recursively lang: painless source: | diff --git a/packages/panw_cortex_xdr/data_stream/alerts/elasticsearch/ingest_pipeline/v1_pipeline.yml b/packages/panw_cortex_xdr/data_stream/alerts/elasticsearch/ingest_pipeline/v1_pipeline.yml index 81491354cf4..699f4031038 100644 --- a/packages/panw_cortex_xdr/data_stream/alerts/elasticsearch/ingest_pipeline/v1_pipeline.yml +++ b/packages/panw_cortex_xdr/data_stream/alerts/elasticsearch/ingest_pipeline/v1_pipeline.yml @@ -2,6 +2,7 @@ description: Pipeline for Palo Alto Cortext XDR Alerts v1 API. processors: - fingerprint: + tag: fingerprint_9f0eb322 fields: - panw_cortex.xdr.events.event_timestamp - panw_cortex.xdr.events.event_id @@ -10,155 +11,186 @@ processors: target_field: "_id" ignore_missing: true - date: + tag: date_panw_cortex_xdr_events_event_timestamp_a2c59f96 field: panw_cortex.xdr.events.event_timestamp formats: - UNIX_MS if: ctx.panw_cortex?.xdr?.events?.event_timestamp != null - date: + tag: date_panw_cortex_xdr_events_agent_host_boot_time_to_panw_cortex_xdr_events_agent_host_boot_time_8d10f812 field: panw_cortex.xdr.events.agent_host_boot_time target_field: panw_cortex.xdr.events.agent_host_boot_time formats: - UNIX_MS if: ctx.panw_cortex?.xdr?.events?.agent_host_boot_time != null - date: + tag: date_panw_cortex_xdr_detection_timestamp_to_event_created_78ac6df9 field: panw_cortex.xdr.detection_timestamp target_field: event.created formats: - UNIX_MS if: ctx.panw_cortex?.xdr?.detection_timestamp != null - date: + tag: date_panw_cortex_xdr_end_match_attempt_ts_to_panw_cortex_xdr_end_match_attempt_ts_964a05e8 field: panw_cortex.xdr.end_match_attempt_ts target_field: panw_cortex.xdr.end_match_attempt_ts formats: - UNIX_MS if: ctx.panw_cortex?.xdr?.end_match_attempt_ts != null - date: + tag: date_panw_cortex_xdr_local_insert_ts_to_panw_cortex_xdr_local_insert_ts_3f6359ea field: panw_cortex.xdr.local_insert_ts target_field: panw_cortex.xdr.local_insert_ts formats: - UNIX_MS if: ctx.panw_cortex?.xdr?.local_insert_ts != null - rename: + tag: rename_panw_cortex_xdr_name_to_message_2d8d51d6 field: panw_cortex.xdr.name target_field: message ignore_missing: true override: true - set: + tag: set_event_severity_8fa551aa field: event.severity value: 0 if: ctx.panw_cortex?.xdr?.severity == "unknown" - set: + tag: set_event_severity_dc82f1ca field: event.severity value: 1 if: ctx.panw_cortex?.xdr?.severity == "informational" - set: + tag: set_event_severity_4987ec40 field: event.severity value: 2 if: ctx.panw_cortex?.xdr?.severity == "low" - set: + tag: set_event_severity_337f7362 field: event.severity value: 3 if: ctx.panw_cortex?.xdr?.severity == "medium" - set: + tag: set_event_severity_a870e6fa field: event.severity value: 4 if: ctx.panw_cortex?.xdr?.severity == "high" - rename: + tag: rename_panw_cortex_xdr_external_id_to_event_id_c86f2ab4 field: panw_cortex.xdr.external_id target_field: event.id ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_action_to_event_action_42c157a6 field: panw_cortex.xdr.action target_field: event.action ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_description_to_event_reason_91664e45 field: panw_cortex.xdr.description target_field: event.reason ignore_missing: true if: ctx.panw_cortex?.xdr?.description instanceof String override: true - rename: + tag: rename_panw_cortex_xdr_description_to_panw_cortex_xdr_bioc_description_e00626c5 field: panw_cortex.xdr.description target_field: panw_cortex.xdr.bioc_description ignore_missing: true if: ctx.event?.reason == null && ctx.panw_cortex?.xdr?.description instanceof List override: true - set: + tag: set_event_reason_fd9a9809 field: event.reason value: Bioc Event if: ctx.event?.reason == null && ctx.panw_cortex?.xdr?.bioc_description != null - rename: + tag: rename_panw_cortex_xdr_agent_device_domain_to_host_domain_54c9210f field: panw_cortex.xdr.agent_device_domain target_field: host.domain ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_agent_fqdn_to_host_hostname_d1325b5e field: panw_cortex.xdr.agent_fqdn target_field: host.hostname ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_host_name_to_host_hostname_b3c2856a field: panw_cortex.xdr.host_name target_field: host.hostname ignore_missing: true if: ctx.host?.hostname == null override: true - lowercase: + tag: lowercase_host_hostname_to_host_name_39c0d2d1 target_field: host.name field: host.hostname if: ctx.host?.hostname != null - rename: + tag: rename_panw_cortex_xdr_agent_os_type_to_host_os_name_2a107338 field: panw_cortex.xdr.agent_os_type target_field: host.os.name ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_agent_os_sub_type_to_host_os_version_09b35710 field: panw_cortex.xdr.agent_os_sub_type target_field: host.os.version ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_mac_addresses_to_host_mac_71ef9ecf field: panw_cortex.xdr.mac_addresses target_field: host.mac ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_host_ip_to_host_ip_727babf7 field: panw_cortex.xdr.host_ip target_field: host.ip ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_endpoint_id_to_host_id_0ddf3a76 field: panw_cortex.xdr.endpoint_id target_field: host.id ignore_missing: true override: true - split: + tag: split_panw_cortex_xdr_mac_to_host_mac_25564f0c field: panw_cortex.xdr.mac target_field: host.mac separator: "," ignore_missing: true if: ctx.host?.mac == null - remove: + tag: remove_c40d8f42 field: - panw_cortex.xdr.mac ignore_missing: true if: ctx.host?.mac != null - gsub: + tag: gsub_host_mac_327602f7 field: host.mac ignore_missing: true pattern: '[:.]' replacement: '-' - uppercase: + tag: uppercase_host_mac_73cd4413 field: host.mac ignore_missing: true - rename: + tag: rename_panw_cortex_xdr_events_dns_query_name_to_dns_question_name_ac90fe5d field: panw_cortex.xdr.events.dns_query_name target_field: dns.question.name ignore_missing: true override: true - script: + tag: script_ad7409b5 if: ctx.panw_cortex?.xdr?.mitre_technique_id_and_name != null lang: painless description: "Extract Mitre Techniques and append it to Threat ECS fields" @@ -187,6 +219,7 @@ processors: addTechnique(ctx, mitre_technique.splitOnToken(' - ')[0], mitre_technique.splitOnToken(' - ')[1]); } - script: + tag: script_45d60717 if: ctx.panw_cortex?.xdr?.mitre_tactic_id_and_name != null lang: painless description: "Extract Mitre Tactics and append it to Threat ECS fields" @@ -215,204 +248,243 @@ processors: addTactic(ctx, mitre_tactic.splitOnToken(' - ')[0], mitre_tactic.splitOnToken(' - ')[1]); } - set: + tag: set_threat_framework_5b18ca5e field: threat.framework value: "MITRE ATT&CK" if: ctx.threat?.technique != null || ctx.threat?.tactic != null #The Action actor is an an activity that took place and was recorded by the agent. - convert: + tag: convert_panw_cortex_xdr_events_action_remote_ip_to_destination_ip_5df66cda field: panw_cortex.xdr.events.action_remote_ip target_field: destination.ip type: ip ignore_missing: true ignore_failure: true - convert: + tag: convert_panw_cortex_xdr_events_action_remote_port_to_destination_port_907e1efe field: panw_cortex.xdr.events.action_remote_port target_field: destination.port type: long ignore_missing: true - convert: + tag: convert_panw_cortex_xdr_events_action_local_ip_to_source_ip_51931217 field: panw_cortex.xdr.events.action_local_ip target_field: source.ip type: ip ignore_missing: true - convert: + tag: convert_panw_cortex_xdr_events_action_local_port_to_source_port_6b2c0f42 field: panw_cortex.xdr.events.action_local_port target_field: source.port type: long ignore_missing: true - rename: + tag: rename_panw_cortex_xdr_events_action_process_image_sha256_to_process_hash_sha256_f8e3b3f5 field: panw_cortex.xdr.events.action_process_image_sha256 target_field: process.hash.sha256 ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_action_process_image_command_line_to_process_command_line_986cd85b field: panw_cortex.xdr.events.action_process_image_command_line target_field: process.command_line ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_action_process_image_name_to_process_name_c279881b field: panw_cortex.xdr.events.action_process_image_name target_field: process.name ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_action_process_signature_vendor_to_process_code_signature_subject_name_0bb860be field: panw_cortex.xdr.events.action_process_signature_vendor target_field: process.code_signature.subject_name ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_action_process_signature_status_to_process_code_signature_status_3604825e field: panw_cortex.xdr.events.action_process_signature_status target_field: process.code_signature.status ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_action_process_instance_id_to_process_entity_id_9ad604bb field: panw_cortex.xdr.events.action_process_instance_id target_field: process.entity_id ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_action_file_path_to_file_path_ffc488c7 field: panw_cortex.xdr.events.action_file_path target_field: file.path ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_action_file_name_to_file_name_7201d1e7 field: panw_cortex.xdr.events.action_file_name target_field: file.name ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_action_file_md5_to_file_hash_md5_3b34151f field: panw_cortex.xdr.events.action_file_md5 target_field: file.hash.md5 ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_action_file_sha256_to_file_hash_sha256_abc0c211 field: panw_cortex.xdr.events.action_file_sha256 target_field: file.hash.sha256 ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_action_registry_key_name_to_registry_key_1b911741 field: panw_cortex.xdr.events.action_registry_key_name target_field: registry.key ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_action_registry_value_name_to_registry_value_ee5d0075 field: panw_cortex.xdr.events.action_registry_value_name target_field: registry.value ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_action_registry_full_key_to_registry_path_36834cff field: panw_cortex.xdr.events.action_registry_full_key target_field: registry.path ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_action_registry_data_to_registry_data_strings_827a315b field: panw_cortex.xdr.events.action_registry_data target_field: registry.data.strings ignore_missing: true override: true - set: + tag: set_registry_data_strings_9dc338e0 field: registry.data.strings value: ['{{{registry.data.strings}}}'] if: ctx.registry?.data?.strings instanceof String #The Actor actor is the process that performed the action. - rename: + tag: rename_panw_cortex_xdr_events_actor_process_os_pid_to_process_pid_02069895 field: panw_cortex.xdr.events.actor_process_os_pid target_field: process.pid ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_actor_process_instance_id_to_process_entity_id_187391c0 field: panw_cortex.xdr.events.actor_process_instance_id target_field: process.entity_id ignore_missing: true if: ctx.process?.entity_id == null override: true - rename: + tag: rename_panw_cortex_xdr_events_actor_process_image_path_to_process_executable_e160d5f1 field: panw_cortex.xdr.events.actor_process_image_path target_field: process.executable ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_actor_process_command_line_to_process_command_line_0cadc0e1 field: panw_cortex.xdr.events.actor_process_command_line target_field: process.command_line ignore_missing: true if: ctx.process?.command_line == null override: true - rename: + tag: rename_panw_cortex_xdr_events_actor_process_image_name_to_process_name_f35d7b6c field: panw_cortex.xdr.events.actor_process_image_name target_field: process.name ignore_missing: true if: ctx.process?.name == null override: true - rename: + tag: rename_panw_cortex_xdr_events_actor_process_signature_vendor_to_process_code_signature_subject_name_fc687cab field: panw_cortex.xdr.events.actor_process_signature_vendor target_field: process.code_signature.subject_name ignore_missing: true if: ctx.process?.code_signature?.subject_name == null override: true - rename: + tag: rename_panw_cortex_xdr_events_actor_process_image_sha256_to_process_hash_sha256_6ffeeef9 field: panw_cortex.xdr.events.actor_process_image_sha256 target_field: process.hash.sha256 ignore_missing: true if: ctx.process?.hash?.sha256 == null override: true - rename: + tag: rename_panw_cortex_xdr_events_actor_process_image_md5_to_process_hash_md5_ba7c4c5e field: panw_cortex.xdr.events.actor_process_image_md5 target_field: process.hash.md5 ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_actor_thread_thread_id_to_process_thread_id_861318f6 field: panw_cortex.xdr.events.actor_thread_thread_id target_field: process.thread.id ignore_missing: true override: true #The Causality actor—also referred to as the causality group owner (CGO)—is the parent process in the execution chain that the Cortex XDR agent identified as being responsible for initiating the process tree. - rename: + tag: rename_panw_cortex_xdr_events_causality_actor_process_image_name_to_process_parent_name_c50d3c44 field: panw_cortex.xdr.events.causality_actor_process_image_name target_field: process.parent.name ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_causality_actor_process_image_path_to_process_parent_executable_ddda3f75 field: panw_cortex.xdr.events.causality_actor_process_image_path target_field: process.parent.executable ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_causality_actor_process_image_md5_to_process_parent_hash_md5_16780578 field: panw_cortex.xdr.events.causality_actor_process_image_md5 target_field: process.parent.hash.md5 ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_causality_actor_process_image_sha256_to_process_parent_hash_sha256_0ecdc3c8 field: panw_cortex.xdr.events.causality_actor_process_image_sha256 target_field: process.parent.hash.sha256 ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_causality_actor_causality_id_to_process_parent_entity_id_c6e298da field: panw_cortex.xdr.events.causality_actor_causality_id target_field: process.parent.entity_id ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_causality_actor_process_signature_vendor_to_process_parent_code_signature_subject_name_37549a0f field: panw_cortex.xdr.events.causality_actor_process_signature_vendor target_field: process.parent.code_signature.subject_name ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_causality_actor_process_signature_status_to_process_parent_code_signature_status_5187943f field: panw_cortex.xdr.events.causality_actor_process_signature_status target_field: process.parent.code_signature.status ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_causality_actor_process_command_line_to_process_parent_command_line_f5f09928 field: panw_cortex.xdr.events.causality_actor_process_command_line target_field: process.parent.command_line ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_causality_actor_process_execution_time_to_process_parent_uptime_839d5b56 field: panw_cortex.xdr.events.causality_actor_process_execution_time target_field: process.parent.uptime ignore_missing: true override: true # Parse user.domain from user.name - grok: + tag: grok_panw_cortex_xdr_events_user_name_474a7106 field: panw_cortex.xdr.events.user_name ignore_missing: true patterns: @@ -422,62 +494,76 @@ processors: - '^%{DATA:user.name}$' if: ctx.panw_cortex?.xdr?.events?.user_name != null - set: + tag: set_user_email_51aa4ed6 field: user.email copy_from: panw_cortex.xdr.events.user_name if: ctx.panw_cortex?.xdr?.events?.user_name instanceof String && ctx.panw_cortex.xdr.events.user_name.contains('@') && ctx.panw_cortex.xdr.events.user_name.contains('.') - set: + tag: set_user_id_782efceb field: user.id copy_from: panw_cortex.xdr.events.user_name if: ctx.panw_cortex?.xdr?.events?.user_name instanceof String && ctx.panw_cortex.xdr.events.user_name.contains('@') && ctx.panw_cortex.xdr.events.user_name.contains('.') - remove: + tag: remove_panw_cortex_xdr_events_user_name_8cafe04c field: panw_cortex.xdr.events.user_name ignore_missing: true - rename: + tag: rename_panw_cortex_xdr_events_fw_rule_to_rule_name_1b830c24 field: panw_cortex.xdr.events.fw_rule target_field: rule.name ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_fw_rule_id_to_rule_id_1c501182 field: panw_cortex.xdr.events.fw_rule_id target_field: rule.id ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_fw_interface_from_to_observer_ingress_interface_name_2d165142 field: panw_cortex.xdr.events.fw_interface_from target_field: observer.ingress.interface.name ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_fw_interface_to_to_observer_egress_interface_name_ae325773 field: panw_cortex.xdr.events.fw_interface_to target_field: observer.egress.interface.name ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_events_fw_serial_number_to_observer_serial_number_f889a9e9 field: panw_cortex.xdr.events.fw_serial_number target_field: observer.serial_number ignore_missing: true override: true - set: + tag: set_email_subject_44ae94ee field: email.subject copy_from: panw_cortex.xdr.events.fw_email_subject if: ctx.panw_cortex.xdr?.events?.fw_email_subject != null - append: + tag: append_email_from_address_9e710573 field: email.from.address value: "{{{panw_cortex.xdr.events.fw_email_sender}}}" if: ctx.panw_cortex.xdr?.events?.fw_email_sender != null - append: + tag: append_email_to_address_4999ea3a field: email.to.address value: "{{{panw_cortex.xdr.events.fw_email_recipient}}}" if: ctx.panw_cortex.xdr?.events?.fw_email_recipient != null - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_geo_ab5e2968 field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -486,6 +572,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -494,61 +581,73 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_e314b953 field: source.as.asn target_field: source.as.number ignore_missing: true override: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_86ec1065 field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true override: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_5217ce8f field: destination.as.asn target_field: destination.as.number ignore_missing: true override: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_13979e17 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true override: true - append: + tag: append_related_hash_badcbdc2 field: related.hash value: "{{{process.parent.hash.md5}}}" allow_duplicates: false if: ctx.process?.parent?.hash?.md5 != null - append: + tag: append_related_hash_fd8e170a field: related.hash value: "{{{process.parent.hash.sha256}}}" allow_duplicates: false if: ctx.process?.parent?.hash?.sha256 != null - append: + tag: append_related_hash_e71b2329 field: related.hash value: "{{{process.hash.md5}}}" allow_duplicates: false if: ctx.process?.hash?.md5 != null - append: + tag: append_related_hash_239dcc07 field: related.hash value: "{{{process.hash.sha256}}}" allow_duplicates: false if: ctx.process?.hash?.sha256 != null - append: + tag: append_related_hash_017aef51 field: related.hash value: "{{{file.hash.sha256}}}" allow_duplicates: false if: ctx.file?.hash?.sha256 != null - append: + tag: append_related_hash_6af42da3 field: related.hash value: "{{{file.hash.md5}}}" allow_duplicates: false if: ctx.file?.hash?.md5 != null - append: + tag: append_related_user_837e080f field: related.user value: "{{{user.name}}}" allow_duplicates: false if: ctx.user?.name != null - foreach: + tag: foreach_panw_cortex_xdr_tags_071d6bfe field: panw_cortex.xdr.tags if: ctx.panw_cortex?.xdr?.tags != null processor: @@ -557,6 +656,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - remove: + tag: remove_70fa3a05 field: - panw_cortex.xdr.host_name - panw_cortex.xdr.detection_timestamp @@ -581,3 +681,7 @@ on_failure: - append: field: error.message value: '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/panw_cortex_xdr/data_stream/alerts/elasticsearch/ingest_pipeline/v2_pipeline.yml b/packages/panw_cortex_xdr/data_stream/alerts/elasticsearch/ingest_pipeline/v2_pipeline.yml index 76bbd7ac6a7..c079b32bc67 100644 --- a/packages/panw_cortex_xdr/data_stream/alerts/elasticsearch/ingest_pipeline/v2_pipeline.yml +++ b/packages/panw_cortex_xdr/data_stream/alerts/elasticsearch/ingest_pipeline/v2_pipeline.yml @@ -18,9 +18,11 @@ processors: if: ctx.panw_cortex?.xdr?.event_timestamp instanceof List on_failure: - remove: + tag: remove_panw_cortex_xdr_event_timestamp_873a36be field: panw_cortex.xdr.event_timestamp ignore_missing: true - append: + tag: append_error_message_2c4b9cc2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -32,9 +34,11 @@ processors: if: ctx.panw_cortex?.xdr?.agent_host_boot_time instanceof List on_failure: - remove: + tag: remove_panw_cortex_xdr_agent_host_boot_time_c726178b field: panw_cortex.xdr.agent_host_boot_time ignore_missing: true - append: + tag: append_error_message_787e6b71 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -46,9 +50,11 @@ processors: if: ctx.panw_cortex?.xdr?.detection_timestamp != null on_failure: - remove: + tag: remove_panw_cortex_xdr_detection_timestamp_9780a4b0 field: panw_cortex.xdr.detection_timestamp ignore_missing: true - append: + tag: append_error_message_ac1fa2e9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -65,9 +71,11 @@ processors: if: ctx.panw_cortex?.xdr?.end_match_attempt_ts != null on_failure: - remove: + tag: remove_panw_cortex_xdr_end_match_attempt_ts_235913e4 field: panw_cortex.xdr.end_match_attempt_ts ignore_missing: true - append: + tag: append_error_message_117fa0be field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -79,9 +87,11 @@ processors: if: ctx.panw_cortex?.xdr?.local_insert_ts != null on_failure: - remove: + tag: remove_panw_cortex_xdr_local_insert_ts_a773940c field: panw_cortex.xdr.local_insert_ts ignore_missing: true - append: + tag: append_error_message_1ccee4f0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -93,9 +103,11 @@ processors: if: ctx.panw_cortex?.xdr?.last_modified_ts != null on_failure: - remove: + tag: remove_panw_cortex_xdr_last_modified_ts_a25ee95a field: panw_cortex.xdr.last_modified_ts ignore_missing: true - append: + tag: append_error_message_01e1c54d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -146,7 +158,7 @@ processors: if: ctx.panw_cortex?.xdr?.description instanceof String - rename: field: panw_cortex.xdr.description - tag: rename_panw_cortex.xdr.description + tag: rename_panw_cortex_xdr_description_to_panw_cortex_xdr_bioc_description_fc018968 target_field: panw_cortex.xdr.bioc_description ignore_missing: true if: ctx.event?.reason == null && ctx.panw_cortex?.xdr?.description instanceof List @@ -229,9 +241,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_panw_cortex_xdr_agent_ip_addresses_v6_1ae3d7c5 field: panw_cortex.xdr.agent_ip_addresses_v6 ignore_missing: true - append: + tag: append_error_message_76e1dacf field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -261,9 +275,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_panw_cortex_xdr_agent_data_collection_status_81e42821 field: panw_cortex.xdr.agent_data_collection_status ignore_missing: true - append: + tag: append_error_message_364af604 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -274,9 +290,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_panw_cortex_xdr_agent_is_vdi_22a8b137 field: panw_cortex.xdr.agent_is_vdi ignore_missing: true - append: + tag: append_error_message_ac1925a8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -365,9 +383,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_panw_cortex_xdr_case_id_9c28d44d field: panw_cortex.xdr.case_id ignore_missing: true - append: + tag: append_error_message_ddd69372 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' #The Action actor is an an activity that took place and was recorded by the agent. @@ -379,9 +399,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_panw_cortex_xdr_action_remote_ip_126c0f01 field: panw_cortex.xdr.action_remote_ip ignore_missing: true - append: + tag: append_error_message_bc4038bf field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -407,9 +429,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_panw_cortex_xdr_action_remote_ip_v6_5698c404 field: panw_cortex.xdr.action_remote_ip_v6 ignore_missing: true - append: + tag: append_error_message_5b95d1ef field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -435,9 +459,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_panw_cortex_xdr_action_remote_port_dfa57b6a field: panw_cortex.xdr.action_remote_port ignore_missing: true - append: + tag: append_error_message_ce6af14a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -447,6 +473,7 @@ processors: tag: set_destination.port on_failure: - append: + tag: append_error_message_d69507cd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -457,9 +484,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_panw_cortex_xdr_action_local_ip_b5bbab76 field: panw_cortex.xdr.action_local_ip ignore_missing: true - append: + tag: append_error_message_527d90ef field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -485,9 +514,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_panw_cortex_xdr_action_local_ip_v6_b9282255 field: panw_cortex.xdr.action_local_ip_v6 ignore_missing: true - append: + tag: append_error_message_f0c15edb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -513,9 +544,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_panw_cortex_xdr_action_local_port_8c4f5fcd field: panw_cortex.xdr.action_local_port ignore_missing: true - append: + tag: append_error_message_e96001f2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -525,6 +558,7 @@ processors: tag: set_source.port on_failure: - append: + tag: append_error_message_54ecabf4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -759,9 +793,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_panw_cortex_xdr_filter_rule_id_20285b12 field: panw_cortex.xdr.filter_rule_id ignore_missing: true - append: + tag: append_error_message_e2739584 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -873,7 +909,7 @@ processors: allow_duplicates: false - append: field: related.hash - tag: append_related.hash + tag: append_related_hash_696669ee value: "{{{process.parent.hash.sha256}}}" allow_duplicates: false if: ctx.process?.parent?.hash?.sha256 instanceof String @@ -888,7 +924,7 @@ processors: allow_duplicates: false - append: field: related.hash - tag: append_related.hash + tag: append_related_hash_22d45925 value: "{{{process.hash.md5}}}" allow_duplicates: false if: ctx.process?.hash?.md5 instanceof String @@ -903,7 +939,7 @@ processors: allow_duplicates: false - append: field: related.hash - tag: append_related.hash + tag: append_related_hash_5aec441f value: "{{{process.hash.sha256}}}" allow_duplicates: false if: ctx.process?.hash?.sha256 instanceof String @@ -918,7 +954,7 @@ processors: allow_duplicates: false - append: field: related.hash - tag: append_related.hash + tag: append_related_hash_6bd5a1cd value: "{{{file.hash.sha256}}}" allow_duplicates: false if: ctx.file?.hash?.sha256 instanceof String @@ -933,7 +969,7 @@ processors: allow_duplicates: false - append: field: related.hash - tag: append_related.hash + tag: append_related_hash_3c3aa583 value: "{{{file.hash.md5}}}" allow_duplicates: false if: ctx.file?.hash?.md5 != null @@ -967,6 +1003,7 @@ processors: ignore_missing: true tag: remove_fields - remove: + tag: remove_dadde6a8 field: - panw_cortex.xdr.detection_timestamp - panw_cortex.xdr.event_timestamp @@ -1000,3 +1037,7 @@ on_failure: Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/panw_cortex_xdr/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/panw_cortex_xdr/data_stream/event/elasticsearch/ingest_pipeline/default.yml index 7378f7bbfb2..c53e8711c1c 100644 --- a/packages/panw_cortex_xdr/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/panw_cortex_xdr/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -51,6 +51,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -83,6 +84,7 @@ processors: if: ctx.json?.action_boot_time != null && ctx.json.action_boot_time != '' on_failure: - append: + tag: append_error_message_86540155 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -94,6 +96,7 @@ processors: if: ctx.json?.action_file_access_time != null && ctx.json.action_file_access_time != '' on_failure: - append: + tag: append_error_message_2eee35f1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -121,6 +124,7 @@ processors: if: ctx.json?.action_file_create_time != null && ctx.json.action_file_create_time != '' on_failure: - append: + tag: append_error_message_248c9567 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -161,6 +165,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_058c3d02 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -193,6 +198,7 @@ processors: if: ctx.json?.action_file_mod_time != null && ctx.json.action_file_mod_time != '' on_failure: - append: + tag: append_error_message_910a32d1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -309,6 +315,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9c92876c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -339,6 +346,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d8ac35bd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -464,6 +472,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_678fac91 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -521,6 +530,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_34bbadde field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -556,6 +566,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_68f4158a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -572,6 +583,7 @@ processors: if: ctx.json?.action_network_creation_time != null && ctx.json.action_network_creation_time != '' on_failure: - append: + tag: append_error_message_9dd275ac field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -591,6 +603,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9e32c5f7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -663,6 +676,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_64993b99 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -674,6 +688,7 @@ processors: if: ctx.json?.action_process_instance_execution_time != null && ctx.json.action_process_instance_execution_time != '' on_failure: - append: + tag: append_error_message_8bd4fcae field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -699,6 +714,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_b1bb7865 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -744,6 +760,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8c87a68b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -789,6 +806,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4104a6af field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -805,6 +823,7 @@ processors: if: ctx.json?.action_process_termination_date != null && ctx.json.action_process_termination_date != '' on_failure: - append: + tag: append_error_message_2af04609 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1045,6 +1064,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_1a644186 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1060,6 +1080,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8e02c2ca field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1132,6 +1153,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e9e81fcb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1142,6 +1164,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ff9188e8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1152,6 +1175,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e0baec53 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1364,6 +1388,7 @@ processors: if: ctx.event?.action != '' on_failure: - append: + tag: append_error_message_ce1e3af4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - join: @@ -1373,6 +1398,7 @@ processors: if: ctx.event?.action != null on_failure: - append: + tag: append_error_message_f136491d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1459,6 +1485,7 @@ processors: ctx.host.put("name", hostname); on_failure: - append: + tag: append_error_message_6a80d8c1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -1481,6 +1508,7 @@ processors: if: ctx.json?.os_actor_local_ip != '' on_failure: - append: + tag: append_error_message_31f60386 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -1503,6 +1531,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_33e84f01 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -1623,6 +1652,7 @@ processors: if: ctx.json?.os_actor_remote_ip != '' on_failure: - append: + tag: append_error_message_cb89f04a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -1776,6 +1806,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event if: ctx.error?.message != null diff --git a/packages/panw_cortex_xdr/data_stream/incidents/elasticsearch/ingest_pipeline/default.yml b/packages/panw_cortex_xdr/data_stream/incidents/elasticsearch/ingest_pipeline/default.yml index 7e465b12804..82f5a51df8f 100644 --- a/packages/panw_cortex_xdr/data_stream/incidents/elasticsearch/ingest_pipeline/default.yml +++ b/packages/panw_cortex_xdr/data_stream/incidents/elasticsearch/ingest_pipeline/default.yml @@ -13,6 +13,7 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - rename: + tag: rename_message_to_event_original_a6172163 description: Use 'event.original' if present (e.g. when reprocessing), otherwise 'message'. field: message target_field: event.original @@ -25,25 +26,32 @@ processors: description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - set: + tag: set_event_kind_39295792 field: event.kind value: alert - append: + tag: append_event_category_557b9332 field: event.category value: malware allow_duplicates: false - append: + tag: append_event_type_3f43a39d field: event.type value: info allow_duplicates: false - json: + tag: json_event_original_to_panw_cortex_xdr_8d2b549c field: event.original target_field: panw_cortex.xdr - drop: + tag: drop_c1f77990 if: ctx.panw_cortex?.xdr?.reply?.result_count == 0 - fingerprint: + tag: fingerprint_5573d0b8 fields: - panw_cortex.xdr.creation_time - panw_cortex.xdr.modification_time @@ -51,84 +59,102 @@ processors: target_field: "_id" ignore_missing: true - date: + tag: date_panw_cortex_xdr_creation_time_to_panw_cortex_xdr_creation_time_666fca52 field: panw_cortex.xdr.creation_time target_field: panw_cortex.xdr.creation_time formats: - UNIX_MS if: ctx.panw_cortex?.xdr?.creation_time != null - date: + tag: date_panw_cortex_xdr_resolved_timestamp_to_panw_cortex_xdr_resolved_timestamp_90489354 field: panw_cortex.xdr.resolved_timestamp target_field: panw_cortex.xdr.resolved_timestamp formats: - UNIX_MS if: ctx.panw_cortex?.xdr?.resolved_timestamp != null - date: + tag: date_panw_cortex_xdr_modification_time_to_panw_cortex_xdr_modification_time_92a41899 field: panw_cortex.xdr.modification_time target_field: panw_cortex.xdr.modification_time formats: - UNIX_MS if: ctx.panw_cortex?.xdr?.modification_time != null - set: + tag: set_event_created_f24032fa field: event.created copy_from: panw_cortex.xdr.creation_time - set: + tag: set_timestamp_271c16de field: "@timestamp" copy_from: panw_cortex.xdr.modification_time if: ctx.panw_cortex?.xdr?.modification_time != null - set: + tag: set_event_severity_8fa551aa field: event.severity value: 0 if: ctx.panw_cortex?.xdr?.severity == "unknown" - set: + tag: set_event_severity_dc82f1ca field: event.severity value: 1 if: ctx.panw_cortex?.xdr?.severity == "informational" - set: + tag: set_event_severity_4987ec40 field: event.severity value: 2 if: ctx.panw_cortex?.xdr?.severity == "low" - set: + tag: set_event_severity_337f7362 field: event.severity value: 3 if: ctx.panw_cortex?.xdr?.severity == "medium" - set: + tag: set_event_severity_a870e6fa field: event.severity value: 4 if: ctx.panw_cortex?.xdr?.severity == "high" - set: + tag: set_event_severity_b1bedd26 field: event.severity value: 5 if: ctx.panw_cortex?.xdr?.severity == "critical" - rename: + tag: rename_panw_cortex_xdr_incident_id_to_event_id_cad292d1 field: panw_cortex.xdr.incident_id target_field: event.id ignore_missing: true override: true - rename: + tag: rename_panw_cortex_xdr_description_to_event_reason_91664e45 field: panw_cortex.xdr.description target_field: event.reason ignore_missing: true if: ctx.panw_cortex?.xdr?.description instanceof String override: true - set: + tag: set_related_hosts_7b889fc4 field: related.hosts copy_from: panw_cortex.xdr.hosts if: ctx.panw_cortex?.xdr?.hosts != null - set: + tag: set_related_user_4f2c73e6 field: related.user copy_from: panw_cortex.xdr.users if: ctx.panw_cortex?.xdr?.users != null - gsub: + tag: gsub_related_hosts_8c0d27ac field: related.hosts pattern: ":.*" replacement: "" if: ctx.related?.hosts != null - gsub: + tag: gsub_related_user_caf99d20 field: related.user pattern: '.*\\' replacement: "" if: ctx.related?.user != null - script: + tag: script_93c0aaf9 if: ctx.panw_cortex?.xdr?.mitre_techniques_ids_and_names != null lang: painless description: "Extract Mitre Techniques and append it to Threat ECS fields" @@ -157,6 +183,7 @@ processors: addTechnique(ctx, mitre_technique.splitOnToken(' - ')[0], mitre_technique.splitOnToken(' - ')[1]); } - script: + tag: script_98eae421 if: ctx.panw_cortex?.xdr?.mitre_tactics_ids_and_names != null lang: painless description: "Extract Mitre Tactics and append it to Threat ECS fields" @@ -185,20 +212,24 @@ processors: addTactic(ctx, mitre_tactic.splitOnToken(' - ')[0], mitre_tactic.splitOnToken(' - ')[1]); } - set: + tag: set_threat_framework_5b18ca5e field: threat.framework value: "MITRE ATT&CK" if: "ctx.threat?.technique != null || ctx.threat?.tactic != null" - append: + tag: append_related_user_837e080f field: related.user value: "{{{user.name}}}" allow_duplicates: false if: ctx.user?.name != null - append: + tag: append_related_host_979ac57c field: related.host value: "{{{host.name}}}" allow_duplicates: false if: ctx.host?.name != null - foreach: + tag: foreach_panw_cortex_xdr_tags_071d6bfe field: panw_cortex.xdr.tags if: ctx.panw_cortex?.xdr?.tags != null processor: @@ -207,6 +238,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - script: + tag: script_7e38081f description: Drops null/empty values recursively lang: painless source: | @@ -224,6 +256,7 @@ processors: } drop(ctx); - remove: + tag: remove_55e6d391 field: - panw_cortex.xdr.severity - panw_cortex.xdr.tags diff --git a/packages/panw_cortex_xdr/manifest.yml b/packages/panw_cortex_xdr/manifest.yml index f71df49997f..d8401a91882 100644 --- a/packages/panw_cortex_xdr/manifest.yml +++ b/packages/panw_cortex_xdr/manifest.yml @@ -1,6 +1,6 @@ name: panw_cortex_xdr title: Palo Alto Cortex XDR -version: "2.7.1" +version: "2.7.2" description: Collect logs from Palo Alto Cortex XDR with Elastic Agent. type: integration format_version: "3.4.0" diff --git a/packages/ping_federate/changelog.yml b/packages/ping_federate/changelog.yml index da36fb65451..6dd41121fe0 100644 --- a/packages/ping_federate/changelog.yml +++ b/packages/ping_federate/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.2.1" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.2.0" changes: - description: Support IPv6 source addresses with brackets in audit data stream. diff --git a/packages/ping_federate/data_stream/admin/elasticsearch/ingest_pipeline/default.yml b/packages/ping_federate/data_stream/admin/elasticsearch/ingest_pipeline/default.yml index b7450bd0a4f..42b6f3d8fe2 100644 --- a/packages/ping_federate/data_stream/admin/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ping_federate/data_stream/admin/elasticsearch/ingest_pipeline/default.yml @@ -33,12 +33,14 @@ processors: description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null - grok: + tag: grok_event_original_30302277 field: event.original patterns: - '%{DATA:ping_federate.admin.timestamp}\s\|\s%{WORD:ping_federate.admin.user}\s\|\s(%{DATA:ping_federate.admin.roles})?\s\|\s(%{IP:ping_federate.admin.ip})?\s\|\s(%{DATA:ping_federate.admin.event.detail_id})?\s\|\s(%{WORD:ping_federate.admin.component})\s\|\s(%{WORD:ping_federate.admin.event.type})?\s\|\s(%{GREEDYDATA:ping_federate.admin.message})?' if: ctx.event?.original != null on_failure: - append: + tag: append_error_message_7de55181 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -147,13 +149,15 @@ processors: if: ctx.ping_federate?.admin?.timestamp != null && ctx.ping_federate.admin.timestamp != '' on_failure: - remove: + tag: remove_ping_federate_admin_timestamp_29eed34e field: ping_federate.admin.timestamp - append: + tag: append_error_message_9d38946c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' -# The timezone doesn't function as expected with custom date formats like yyyy-MM-dd HH:mm:ss,SSS, -# Hence, first converted the date to ISO8601 format and then applied the event.timezone to the ISO8601 formatted date. -# An issue has been raised for this scenario here: https://github.com/elastic/beats/issues/42133 + # The timezone doesn't function as expected with custom date formats like yyyy-MM-dd HH:mm:ss,SSS, + # Hence, first converted the date to ISO8601 format and then applied the event.timezone to the ISO8601 formatted date. + # An issue has been raised for this scenario here: https://github.com/elastic/beats/issues/42133 - date: field: ping_federate.admin.timestamp tag: date_ping_federate_admin_timestamp_timezone @@ -164,11 +168,14 @@ processors: if: ctx.ping_federate?.admin?.timestamp != null && ctx.ping_federate.admin.timestamp != '' && ctx.event?.timezone != null on_failure: - remove: + tag: remove_ping_federate_admin_timestamp_edf239f6 field: ping_federate.admin.timestamp - append: + tag: append_error_message_e5adb770 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_timestamp_0a0f3bb7 field: '@timestamp' copy_from: ping_federate.admin.timestamp ignore_empty_value: true @@ -190,11 +197,14 @@ processors: separator: ',' on_failure: - remove: + tag: remove_ping_federate_admin_roles_1f216fef field: ping_federate.admin.roles - append: + tag: append_error_message_ac809f1a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_ping_federate_admin_roles_15a7950a field: ping_federate.admin.roles if: ctx.ping_federate?.admin?.roles instanceof List ignore_failure: true @@ -238,8 +248,10 @@ processors: if: ctx.ping_federate?.admin?.ip != '' on_failure: - remove: + tag: remove_ping_federate_admin_ip_e3ad4d87 field: ping_federate.admin.ip - append: + tag: append_error_message_08d0e5aa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: diff --git a/packages/ping_federate/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/ping_federate/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 8ee9d03d247..43d1965d527 100644 --- a/packages/ping_federate/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ping_federate/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -110,8 +110,10 @@ processors: if: ctx.ping_federate?.audit?.severity != '' on_failure: - remove: + tag: remove_ping_federate_audit_severity_3e863faa field: ping_federate.audit.severity - append: + tag: append_error_message_f58e56f9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -158,8 +160,10 @@ processors: if: ctx.cef?.extensions?.deviceReceiptTime != null && ctx.cef.extensions.deviceReceiptTime != '' && ctx.event?.timezone == null on_failure: - remove: + tag: remove_ping_federate_audit_response_time_20b60949 field: ping_federate.audit.response_time - append: + tag: append_error_message_d8c923f2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -172,11 +176,14 @@ processors: if: ctx.cef?.extensions?.deviceReceiptTime != null && ctx.cef.extensions.deviceReceiptTime != '' && ctx.event?.timezone != null on_failure: - remove: + tag: remove_ping_federate_audit_response_time_e6c012d3 field: ping_federate.audit.response_time - append: + tag: append_error_message_63fc07b6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_timestamp_cb3d46d6 field: '@timestamp' copy_from: ping_federate.audit.response_time ignore_empty_value: true @@ -191,6 +198,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_a25582d5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -231,7 +239,7 @@ processors: ignore_empty_value: true - append: field: related.user - tag: append_local_user_id_to_related_user + tag: append_related_user_5a8fbae8 value: '{{{ping_federate.audit.subject}}}' allow_duplicates: false if: ctx.ping_federate?.audit?.subject != null @@ -259,14 +267,12 @@ processors: field: event.outcome tag: set_event_outcome_success value: success - if: ctx.ping_federate?.audit?.status != null && - ctx.ping_federate.audit.status.toLowerCase().contains("success") + if: ctx.ping_federate?.audit?.status != null && ctx.ping_federate.audit.status.toLowerCase().contains("success") - set: field: event.outcome tag: set_event_outcome_failure value: failure - if: ctx.ping_federate?.audit?.status != null && - ctx.ping_federate.audit.status.toLowerCase().contains("fail") + if: ctx.ping_federate?.audit?.status != null && ctx.ping_federate.audit.status.toLowerCase().contains("fail") - set: field: ping_federate.audit.role tag: set_role_from_deviceCustomString4 @@ -279,11 +285,14 @@ processors: separator: ',' on_failure: - remove: + tag: remove_ping_federate_audit_role_6dd3a91d field: ping_federate.audit.role - append: + tag: append_error_message_42e6c8e5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_ping_federate_audit_role_0480d47b field: ping_federate.audit.role if: ctx.ping_federate?.audit?.role instanceof List ignore_failure: true @@ -309,8 +318,10 @@ processors: if: ctx.ping_federate?.audit?.ip != '' on_failure: - remove: + tag: remove_ping_federate_audit_ip_a526a83c field: ping_federate.audit.ip - append: + tag: append_error_message_179b437b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: diff --git a/packages/ping_federate/manifest.yml b/packages/ping_federate/manifest.yml index 0534bae0235..9b6751ff9ff 100644 --- a/packages/ping_federate/manifest.yml +++ b/packages/ping_federate/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.2.1 name: ping_federate title: PingFederate -version: "1.2.0" +version: "1.2.1" description: Collect logs from PingFederate with Elastic Agent. type: integration categories: diff --git a/packages/ping_one/changelog.yml b/packages/ping_one/changelog.yml index 9ef9e14b26f..abc3d12d36b 100644 --- a/packages/ping_one/changelog.yml +++ b/packages/ping_one/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.24.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.24.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/ping_one/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/ping_one/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 432363b7c87..36642b18276 100644 --- a/packages/ping_one/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ping_one/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -2,29 +2,36 @@ description: Pipeline for processing audit logs. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - set: + tag: set_event_kind_de80643c field: event.kind value: event - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_cac66847 field: event.original target_field: json ignore_failure: true - append: + tag: append_event_category_b30667d5 field: event.category value: [iam] - append: + tag: append_event_category_ec4e54a7 field: event.category if: >- ctx.json?.action?.type != null && ( @@ -35,60 +42,74 @@ processors: ) value: [configuration] - append: + tag: append_event_type_d34c7c26 field: event.type if: ctx.json?.action?.type?.toLowerCase()?.contains('created') == true value: [creation] - append: + tag: append_event_type_a6404250 field: event.type if: ctx.json?.action?.type?.toLowerCase()?.contains('deleted') == true value: [deletion] - append: + tag: append_event_type_22235978 field: event.type if: ctx.json?.action?.type?.toLowerCase()?.contains('updated') == true value: [change] - append: + tag: append_event_type_3ba1353f field: event.type if: ctx.json?.action?.type?.toLowerCase()?.contains('user') == true value: [user] - append: + tag: append_event_type_6fb5b9c5 field: event.type if: ctx.json?.action?.type?.toLowerCase()?.contains('group') == true value: [group] - append: + tag: append_event_type_70d7d723 field: event.type if: ctx.json?.action?.type?.toLowerCase()?.contains('allowed') == true value: [info] - append: + tag: append_event_type_d568ebbb field: event.type if: ctx.json?.action?.type?.toLowerCase()?.contains('denied') == true value: [denied] - append: + tag: append_event_type_1ed89a58 field: event.type if: ctx.json?.action?.type?.toLowerCase()?.contains('started') == true value: [start] - append: + tag: append_event_type_c0f55558 field: event.type if: ctx.json?.action?.type?.toLowerCase()?.contains('access_allowed') == true value: [access] - append: + tag: append_event_category_537a9202 field: event.category if: ctx.json?.action?.type?.toLowerCase()?.contains('password.check_succeeded') == true value: [authentication] - append: + tag: append_event_category_c07db32b field: event.category if: ctx.json?.action?.type?.toLowerCase()?.contains('email') == true value: [email] - set: + tag: set_event_type_e4a41200 field: event.type if: ctx.event?.type == null value: [info] - fingerprint: + tag: fingerprint_629464c5 fields: - json.recordedAt - json.id target_field: _id ignore_missing: true - date: + tag: date_json_createdAt_to_ping_one_audit_created_at_a0585be1 field: json.createdAt target_field: ping_one.audit.created_at if: ctx.json?.createdAt != null && ctx.json?.createdAt != '' @@ -96,17 +117,21 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_86f232d6 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json__embedded_to_ping_one_audit_embedded_8015058d field: json._embedded target_field: ping_one.audit.embedded ignore_missing: true - rename: + tag: rename_json_tags_to_ping_one_audit_tags_cf474050 field: json.tags target_field: ping_one.audit.tags ignore_missing: true - foreach: + tag: foreach_ping_one_audit_tags_838fbd8c field: ping_one.audit.tags processor: append: @@ -117,14 +142,17 @@ processors: ignore_failure: true if: ctx.ping_one?.audit?.tags != null && ctx.ping_one?.audit?.tags instanceof List - rename: + tag: rename_json_id_to_ping_one_audit_id_17cbfa30 field: json.id target_field: ping_one.audit.id ignore_missing: true - set: + tag: set_event_id_a500aafc field: event.id copy_from: ping_one.audit.id ignore_failure: true - date: + tag: date_json_recordedAt_to_ping_one_audit_recorded_at_0c267c45 field: json.recordedAt target_field: ping_one.audit.recorded_at if: ctx.json?.recordedAt != null && ctx.json?.recordedAt != '' @@ -132,69 +160,85 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_60179876 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_timestamp_ce00435e field: '@timestamp' copy_from: ping_one.audit.recorded_at ignore_failure: true - rename: + tag: rename_json_correlationId_to_ping_one_audit_correlation_id_ba3d2690 field: json.correlationId target_field: ping_one.audit.correlation.id ignore_missing: true - rename: + tag: rename_json_actors_client_id_to_ping_one_audit_actors_client_id_d64ed70a field: json.actors.client.id target_field: ping_one.audit.actors.client.id ignore_missing: true - set: + tag: set_client_user_id_a07a5407 field: client.user.id copy_from: ping_one.audit.actors.client.id ignore_failure: true - rename: + tag: rename_json_actors_client_name_to_ping_one_audit_actors_client_name_964b26f6 field: json.actors.client.name target_field: ping_one.audit.actors.client.name ignore_missing: true - set: + tag: set_client_user_name_0ace514f field: client.user.name copy_from: ping_one.audit.actors.client.name ignore_failure: true - rename: + tag: rename_json_actors_client_environment_id_to_ping_one_audit_actors_client_environment_id_778657dc field: json.actors.client.environment.id target_field: ping_one.audit.actors.client.environment.id ignore_missing: true - rename: + tag: rename_json_actors_client_href_to_ping_one_audit_actors_client_href_8be4422a field: json.actors.client.href target_field: ping_one.audit.actors.client.href ignore_missing: true - rename: + tag: rename_json_actors_client_type_to_ping_one_audit_actors_client_type_6b36ca0c field: json.actors.client.type target_field: ping_one.audit.actors.client.type ignore_missing: true - append: + tag: append_related_user_318d2323 field: related.user value: '{{{client.user.id}}}' if: ctx.client?.user?.id != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_2353f6bb field: related.user value: '{{{client.user.name}}}' if: ctx.client?.user?.name != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_actors_user_id_to_ping_one_audit_actors_user_id_7cefcfda field: json.actors.user.id target_field: ping_one.audit.actors.user.id ignore_missing: true - set: + tag: set_user_id_7d5e659a field: user.id copy_from: ping_one.audit.actors.user.id ignore_failure: true - rename: + tag: rename_json_actors_user_name_to_ping_one_audit_actors_user_name_0ab42ba6 field: json.actors.user.name target_field: ping_one.audit.actors.user.name ignore_missing: true - set: + tag: set_user_name_755c36e6 field: user.name copy_from: ping_one.audit.actors.user.name ignore_failure: true @@ -211,55 +255,67 @@ processors: ignore_failure: true if: ctx.user?.name == null - rename: + tag: rename_json_actors_user_population_id_to_ping_one_audit_actors_user_population_id_65e76afc field: json.actors.user.population.id target_field: ping_one.audit.actors.user.population.id ignore_missing: true - rename: + tag: rename_json_actors_user_environment_id_to_ping_one_audit_actors_user_environment_id_6026c35c field: json.actors.user.environment.id target_field: ping_one.audit.actors.user.environment.id ignore_missing: true - rename: + tag: rename_json_actors_user_href_to_ping_one_audit_actors_user_href_0bd18b62 field: json.actors.user.href target_field: ping_one.audit.actors.user.href ignore_missing: true - rename: + tag: rename_json_actors_user_type_to_ping_one_audit_actors_user_type_b1ddace4 field: json.actors.user.type target_field: ping_one.audit.actors.user.type ignore_missing: true - append: + tag: append_related_user_3b2f7fde field: related.user value: '{{{user.id}}}' if: ctx.user?.id != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_3b423052 field: related.user value: '{{{user.name}}}' if: ctx.user?.name != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_36d4b55a field: related.user value: '{{{user.email}}}' if: ctx.user?.email != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_action_type_to_ping_one_audit_action_type_4a5b9120 field: json.action.type target_field: ping_one.audit.action.type ignore_missing: true - set: + tag: set_event_action_77eae7de field: event.action copy_from: ping_one.audit.action.type ignore_failure: true - lowercase: + tag: lowercase_event_action_9334b869 field: event.action ignore_missing: true - rename: + tag: rename_json_action_description_to_ping_one_audit_action_description_e34edbe6 field: json.action.description target_field: ping_one.audit.action.description ignore_missing: true - foreach: + tag: foreach_json_resources_e4692695 field: json.resources processor: uri_parts: @@ -268,28 +324,34 @@ processors: ignore_failure: true if: ctx.json?.resources != null && ctx.json?.resources instanceof List - rename: + tag: rename_json_resources_to_ping_one_audit_resources_4aed34dc field: json.resources target_field: ping_one.audit.resources ignore_missing: true - rename: + tag: rename_json_result_id_to_ping_one_audit_result_id_fd7e5a60 field: json.result.id target_field: ping_one.audit.result.id ignore_missing: true - rename: + tag: rename_json_result_status_to_ping_one_audit_result_status_b34e4528 field: json.result.status target_field: ping_one.audit.result.status ignore_missing: true - set: + tag: set_event_outcome_97c2c023 field: event.outcome value: success if: ctx.ping_one?.audit?.result?.status == 'SUCCESS' || ctx.ping_one?.audit?.result?.status == 'succeeded' ignore_failure: true - set: + tag: set_event_outcome_91cc7983 field: event.outcome value: failure if: ctx.ping_one?.audit?.result?.status == 'FAILURE' || ctx.ping_one?.audit?.result?.status == 'failed' ignore_failure: true - rename: + tag: rename_json_result_description_to_ping_one_audit_result_description_b8c624d0 field: json.result.description target_field: ping_one.audit.result.description ignore_missing: true @@ -306,6 +368,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_b539951b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -334,11 +397,13 @@ processors: tag: user_agent_source_user_agent ignore_missing: true - remove: + tag: remove_e5a80934 field: - json ignore_missing: true # Remove ping_one.audit fields that are copied into an ECS field. - foreach: + tag: foreach_ping_one_audit_resources_2b5431d7 field: ping_one.audit.resources processor: remove: @@ -348,6 +413,7 @@ processors: ignore_failure: true if: ctx.ping_one?.audit?.resources != null && ctx.ping_one?.audit?.resources instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) - remove: + tag: remove_d4cb182f if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - ping_one.audit.recorded_at @@ -394,6 +460,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/ping_one/manifest.yml b/packages/ping_one/manifest.yml index 670b9453705..44b2b7c3691 100644 --- a/packages/ping_one/manifest.yml +++ b/packages/ping_one/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.3.2" name: ping_one title: PingOne -version: "1.24.1" +version: "1.24.2" description: Collect logs from PingOne with Elastic-Agent. type: integration categories: diff --git a/packages/pps/changelog.yml b/packages/pps/changelog.yml index 53d3f13a5b0..9081a21d34d 100644 --- a/packages/pps/changelog.yml +++ b/packages/pps/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.2.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.2.1" changes: - description: Fix parsing for plain usernames and space-padded dates for syslog messages from Pleasant Password Server. diff --git a/packages/pps/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/pps/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 8f622d02416..e98dd070ad9 100644 --- a/packages/pps/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/pps/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -2,17 +2,21 @@ description: Pipeline for processing Pleasant Password Server logs. processors: - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - set: + tag: set_ecs_version_f5923549 field: ecs.version value: "8.17.0" - set: + tag: set_event_kind_de80643c field: event.kind value: event - grok: + tag: grok_event_original_82d228ac field: event.original patterns: - '^(?:\d+ )?<%{NUMBER:log.syslog.priority:long}>%{SYSLOGTIMESTAMP:event.created}\s+%{NOTSPACE:host.hostname}\s%{DATA}:%{IP:client.ip}\s-\s%{USERNAME:user.name}@%{DATA:user.domain}\s%{DATA}(?(Success)|(Error))\s-\s%{DATA:event.reason}\s-\s%{GREEDYDATA:message}' @@ -20,6 +24,7 @@ processors: - '^(?:\d+ )?<%{NUMBER:log.syslog.priority:long}>%{SYSLOGTIMESTAMP:event.created}\s+%{NOTSPACE:host.domain}\s%{DATA}:%{IP:client.ip}%{DATA}(?(Success)|(Error))\s-\s%{DATA:event.reason}\s-\s%{GREEDYDATA:message}' - '^%{GREEDYDATA:message}$' - grok: + tag: grok_message_4012bce1 field: message ignore_failure: true ignore_missing: true @@ -35,6 +40,7 @@ processors: - notification\s.*>\s(?:from|to)\s<%{DATA:pps.entry.path}>$ - updated\sentry\s<%{DATA:pps.entry.path}>\schanging\sthe\sname\sfrom\s<%{DATA:pps.entry.name}>\sto\s<%{DATA:pps.entry.target.name}> - grok: + tag: grok_message_0dedc2fb field: message ignore_failure: true ignore_missing: true @@ -42,25 +48,30 @@ processors: - \schanging\sthe\susername\sfrom\s<%{DATA:pps.entry.username}>\sto\s<%{DATA:pps.entry.target.username}> # Set the Event Outcome to Lower Case to be ECS Compliant - lowercase: + tag: lowercase_event_outcome_2beacace field: event.outcome if: ctx.event?.outcome != null # Set the Event Outcome to failure if the event.outcome is error - set: + tag: set_event_outcome_f2eb6d5d field: event.outcome value: "failure" if: ctx.event?.outcome == 'error' # Set the Event Outcome to unknown if neither error nor success is present - set: + tag: set_event_outcome_01487e44 field: event.outcome value: "unknown" if: ctx.event?.outcome != 'success' && ctx.event?.outcome != 'failure' - rename: + tag: rename__conf_tz_offset_to_event_timezone_fb30f3b5 field: _conf.tz_offset target_field: event.timezone if: ctx._conf?.tz_offset != null && ctx._conf.tz_offset != 'local' ignore_missing: true ignore_failure: true - date: + tag: date_event_created_to_event_created_0f0de83e field: event.created target_field: "event.created" formats: @@ -69,12 +80,15 @@ processors: if: ctx.event?.timezone != null && ctx.event?.created != null on_failure: - remove: + tag: remove_event_created_9d2e6185 field: event.created ignore_missing: true - append: + tag: append_error_message_d24defef field: error.message value: "{{{_ingest.on_failure_message}}}" - date: + tag: date_event_created_to_event_created_c53b74d6 field: event.created target_field: "event.created" formats: @@ -82,22 +96,27 @@ processors: if: ctx.event?.timezone == null && ctx.event?.created != null on_failure: - remove: + tag: remove_event_created_11d88a85 field: event.created ignore_missing: true - append: + tag: append_error_message_83f9004b field: error.message value: "{{{_ingest.on_failure_message}}}" # Since logstash sets the @timestamp if not present, `override: true` is required to overwrite the value with event timestamp. - set: + tag: set_timestamp_e8302ea8 field: "@timestamp" value: "{{{event.created}}}" if: "ctx.event?.created != null" override: true - set: + tag: set_user_email_1ed63164 field: "user.email" value: "{{{user.name}}}@{{{user.domain}}}" if: ctx.user?.name != null && ctx.user?.domain != null - remove: + tag: remove_e16fd958 field: - _conf - _tmp diff --git a/packages/pps/manifest.yml b/packages/pps/manifest.yml index 333eb88c1a6..42a2ea784ef 100644 --- a/packages/pps/manifest.yml +++ b/packages/pps/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.3 name: pps title: "Pleasant Password Server" -version: "1.2.1" +version: "1.2.2" source: license: "Apache-2.0" description: "Integration for Pleasant Password Server Syslog Messages" diff --git a/packages/prisma_cloud/changelog.yml b/packages/prisma_cloud/changelog.yml index a8372c73caa..66f323a507f 100644 --- a/packages/prisma_cloud/changelog.yml +++ b/packages/prisma_cloud/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "4.2.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "4.2.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/prisma_cloud/data_stream/alert/elasticsearch/ingest_pipeline/default.yml b/packages/prisma_cloud/data_stream/alert/elasticsearch/ingest_pipeline/default.yml index 296a42f3d05..bc5859e1b99 100644 --- a/packages/prisma_cloud/data_stream/alert/elasticsearch/ingest_pipeline/default.yml +++ b/packages/prisma_cloud/data_stream/alert/elasticsearch/ingest_pipeline/default.yml @@ -35,11 +35,13 @@ processors: ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - drop: + tag: drop_41e92579 if: ctx.event?.original != null && ctx.event.original.isEmpty() - json: field: event.original @@ -47,9 +49,11 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_a6e0e953 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - fingerprint: + tag: fingerprint_b06962fb fields: - json.id - json.lastUpdated @@ -76,6 +80,7 @@ processors: target_field: prisma_cloud.alert.attribution.event_list ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_attribution_event_list_3c03235c field: prisma_cloud.alert.attribution.event_list if: ctx.prisma_cloud?.alert?.attribution?.event_list instanceof List ignore_failure: true @@ -86,6 +91,7 @@ processors: target_field: _ingest._value.value ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_attribution_event_list_bbe8c5c3 field: prisma_cloud.alert.attribution.event_list if: ctx.prisma_cloud?.alert?.attribution?.event_list instanceof List ignore_failure: true @@ -100,6 +106,7 @@ processors: field: _ingest._value.event_ts ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_attribution_event_list_37e26abf field: prisma_cloud.alert.attribution.event_list if: ctx.prisma_cloud?.alert?.attribution?.event_list instanceof List ignore_failure: true @@ -110,6 +117,7 @@ processors: value: '{{{_ingest._value.username}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_alert_attribution_event_list_a4b786e4 field: prisma_cloud.alert.attribution.event_list if: ctx.prisma_cloud?.alert?.attribution?.event_list instanceof List ignore_failure: true @@ -133,6 +141,7 @@ processors: if: ctx.prisma_cloud?.alert?.attribution?.resourceCreatedOn != null && ctx.prisma_cloud.alert.attribution.resourceCreatedOn != '' on_failure: - append: + tag: append_error_message_90edef5a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -144,6 +153,7 @@ processors: if: ctx.json?.alertCount != '' on_failure: - append: + tag: append_error_message_0c2b6a2c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -155,6 +165,7 @@ processors: if: ctx.json?.alertTime != null && ctx.json.alertTime != '' on_failure: - append: + tag: append_error_message_dc33fa29 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -176,6 +187,7 @@ processors: if: ctx.json?.dismissalUntilTs != null && ctx.json.dismissalUntilTs != '' on_failure: - append: + tag: append_error_message_a8abfa89 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -192,6 +204,7 @@ processors: if: ctx.json?.eventOccurred != null && ctx.json.eventOccurred != '' on_failure: - append: + tag: append_error_message_560f47b4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -203,6 +216,7 @@ processors: if: ctx.json?.firstSeen != null && ctx.json.firstSeen != '' on_failure: - append: + tag: append_error_message_aebac8a2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -216,6 +230,7 @@ processors: target_field: prisma_cloud.alert.history ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_history_1efa7e16 field: prisma_cloud.alert.history if: ctx.prisma_cloud?.alert?.history instanceof List ignore_failure: true @@ -226,6 +241,7 @@ processors: target_field: _ingest._value.modified_by ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_history_5c64b818 field: prisma_cloud.alert.history if: ctx.prisma_cloud?.alert?.history instanceof List ignore_failure: true @@ -240,6 +256,7 @@ processors: field: _ingest._value.modifiedOn ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_history_32e585d2 field: prisma_cloud.alert.history if: ctx.prisma_cloud?.alert?.history instanceof List ignore_failure: true @@ -268,6 +285,7 @@ processors: if: ctx.json?.lastSeen != null && ctx.json.lastSeen != '' on_failure: - append: + tag: append_error_message_874a0ad9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -283,17 +301,19 @@ processors: if: ctx.json?.lastUpdated != null && ctx.json.lastUpdated != '' on_failure: - append: + tag: append_error_message_0e94af82 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: field: json.lastUpdated - tag: date_lastUpdated + tag: date_json_lastUpdated_to_prisma_cloud_alert_last_updated_721aee6c target_field: prisma_cloud.alert.last.updated formats: - UNIX_MS if: ctx.json?.lastUpdated != null && ctx.json.lastUpdated != '' on_failure: - append: + tag: append_error_message_d668a8e5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -312,6 +332,7 @@ processors: target_field: prisma_cloud.alert.policy.compliance_metadata ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_policy_compliance_metadata_6197d587 field: prisma_cloud.alert.policy.compliance_metadata if: ctx.prisma_cloud?.alert?.policy?.compliance_metadata instanceof List ignore_failure: true @@ -322,6 +343,7 @@ processors: target_field: _ingest._value.compliance_id ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_policy_compliance_metadata_667408fe field: prisma_cloud.alert.policy.compliance_metadata if: ctx.prisma_cloud?.alert?.policy?.compliance_metadata instanceof List ignore_failure: true @@ -340,6 +362,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_alert_policy_compliance_metadata_cb3914ae field: prisma_cloud.alert.policy.compliance_metadata if: ctx.prisma_cloud?.alert?.policy?.compliance_metadata instanceof List ignore_failure: true @@ -350,6 +373,7 @@ processors: target_field: _ingest._value.policy_id ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_policy_compliance_metadata_cd664f71 field: prisma_cloud.alert.policy.compliance_metadata if: ctx.prisma_cloud?.alert?.policy?.compliance_metadata instanceof List ignore_failure: true @@ -360,6 +384,7 @@ processors: target_field: _ingest._value.requirement.description ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_policy_compliance_metadata_beb0e0dc field: prisma_cloud.alert.policy.compliance_metadata if: ctx.prisma_cloud?.alert?.policy?.compliance_metadata instanceof List ignore_failure: true @@ -370,6 +395,7 @@ processors: target_field: _ingest._value.requirement.id ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_policy_compliance_metadata_bfc75a6a field: prisma_cloud.alert.policy.compliance_metadata if: ctx.prisma_cloud?.alert?.policy?.compliance_metadata instanceof List ignore_failure: true @@ -380,6 +406,7 @@ processors: target_field: _ingest._value.requirement.name ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_policy_compliance_metadata_586a421d field: prisma_cloud.alert.policy.compliance_metadata if: ctx.prisma_cloud?.alert?.policy?.compliance_metadata instanceof List ignore_failure: true @@ -390,6 +417,7 @@ processors: target_field: _ingest._value.section.description ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_policy_compliance_metadata_c5d8af94 field: prisma_cloud.alert.policy.compliance_metadata if: ctx.prisma_cloud?.alert?.policy?.compliance_metadata instanceof List ignore_failure: true @@ -400,6 +428,7 @@ processors: target_field: _ingest._value.section.id ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_policy_compliance_metadata_cce22159 field: prisma_cloud.alert.policy.compliance_metadata if: ctx.prisma_cloud?.alert?.policy?.compliance_metadata instanceof List ignore_failure: true @@ -410,6 +439,7 @@ processors: target_field: _ingest._value.section.label ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_policy_compliance_metadata_73ff6bfc field: prisma_cloud.alert.policy.compliance_metadata if: ctx.prisma_cloud?.alert?.policy?.compliance_metadata instanceof List ignore_failure: true @@ -428,6 +458,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_alert_policy_compliance_metadata_3ca99d14 field: prisma_cloud.alert.policy.compliance_metadata if: ctx.prisma_cloud?.alert?.policy?.compliance_metadata instanceof List ignore_failure: true @@ -446,6 +477,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_alert_policy_compliance_metadata_f20f72ba field: prisma_cloud.alert.policy.compliance_metadata if: ctx.prisma_cloud?.alert?.policy?.compliance_metadata instanceof List ignore_failure: true @@ -464,6 +496,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_alert_policy_compliance_metadata_160659ff field: prisma_cloud.alert.policy.compliance_metadata if: ctx.prisma_cloud?.alert?.policy?.compliance_metadata instanceof List ignore_failure: true @@ -474,6 +507,7 @@ processors: target_field: _ingest._value.standard.description ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_policy_compliance_metadata_916502e0 field: prisma_cloud.alert.policy.compliance_metadata if: ctx.prisma_cloud?.alert?.policy?.compliance_metadata instanceof List ignore_failure: true @@ -484,6 +518,7 @@ processors: target_field: _ingest._value.standard.id ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_policy_compliance_metadata_7cfeaad2 field: prisma_cloud.alert.policy.compliance_metadata if: ctx.prisma_cloud?.alert?.policy?.compliance_metadata instanceof List ignore_failure: true @@ -494,6 +529,7 @@ processors: target_field: _ingest._value.standard.name ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_policy_compliance_metadata_4e496ee2 field: prisma_cloud.alert.policy.compliance_metadata if: ctx.prisma_cloud?.alert?.policy?.compliance_metadata instanceof List ignore_failure: true @@ -520,6 +556,7 @@ processors: if: ctx.json?.policy?.createdOn != null && ctx.json.policy.createdOn != '' on_failure: - append: + tag: append_error_message_a60faac7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -531,6 +568,7 @@ processors: if: ctx.json?.policy?.deleted != '' on_failure: - append: + tag: append_error_message_6d4cc8bd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -547,6 +585,7 @@ processors: if: ctx.json?.policy?.enabled != '' on_failure: - append: + tag: append_error_message_035d87ed field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -573,6 +612,7 @@ processors: if: ctx.json?.policy?.lastModifiedOn != null && ctx.json.policy.lastModifiedOn != '' on_failure: - append: + tag: append_error_message_c2fb6161 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -609,6 +649,7 @@ processors: if: ctx.json?.policy?.remediable != '' on_failure: - append: + tag: append_error_message_6039b439 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -630,6 +671,7 @@ processors: if: ctx.json?.policy?.restrictAlertDismissal != '' on_failure: - append: + tag: append_error_message_e3b49250 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -644,7 +686,7 @@ processors: ignore_missing: true - rename: field: json.policy.rule.cloudType - tag: rename_cloudType + tag: rename_json_policy_rule_cloudType_to_prisma_cloud_alert_policy_rule_cloud_type_02ef601e target_field: prisma_cloud.alert.policy.rule.cloud.type ignore_missing: true - rename: @@ -701,6 +743,7 @@ processors: if: ctx.json?.policy?.ruleLastModifiedOn != null && ctx.json.policy.ruleLastModifiedOn != '' on_failure: - append: + tag: append_error_message_d152fc21 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -717,11 +760,12 @@ processors: if: ctx.json?.policy?.systemDefault != '' on_failure: - append: + tag: append_error_message_82625ee3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: field: json.policyId - tag: rename_policyId + tag: rename_json_policyId_to_prisma_cloud_alert_policy_id_58efa8fb target_field: prisma_cloud.alert.policy_id ignore_missing: true - rename: @@ -781,7 +825,7 @@ processors: ignore_empty_value: true - rename: field: prisma_cloud.alert.resource.cloudType - tag: rename_cloudType + tag: rename_prisma_cloud_alert_resource_cloudType_to_prisma_cloud_alert_resource_cloud_type_c43ebf0a target_field: prisma_cloud.alert.resource.cloud.type ignore_missing: true - set: @@ -813,6 +857,7 @@ processors: if: ctx.prisma_cloud?.alert?.resource?.resourceConfigJsonAvailable != '' on_failure: - append: + tag: append_error_message_cbe6541e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -824,6 +869,7 @@ processors: if: ctx.prisma_cloud?.alert?.resource?.resourceDetailsAvailable != '' on_failure: - append: + tag: append_error_message_5866f0c2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -840,11 +886,12 @@ processors: if: ctx.prisma_cloud?.alert?.resource?.resourceTs != null && ctx.prisma_cloud.alert.resource.resourceTs != '' on_failure: - append: + tag: append_error_message_6fb14aed field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: field: prisma_cloud.alert.resource.resourceType - tag: rename_resourceType + tag: rename_prisma_cloud_alert_resource_resourceType_to_prisma_cloud_alert_resource_type_dbbb90ab target_field: prisma_cloud.alert.resource.type ignore_missing: true - rename: @@ -858,6 +905,7 @@ processors: if: ctx.prisma_cloud?.alert?.resource?.url != null on_failure: - append: + tag: append_error_message_4685ce0d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag fail-{{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -871,6 +919,7 @@ processors: target_field: prisma_cloud.alert.risk_detail.policy_scores ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_a90398f6 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -881,6 +930,7 @@ processors: target_field: _ingest._value.cloud_type ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_13a88f4d field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -891,6 +941,7 @@ processors: target_field: _ingest._value.compliance_metadata ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_561e9c28 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -905,6 +956,7 @@ processors: target_field: _ingest._value.compliance_id ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_4ea63eb1 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -927,6 +979,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_e719ebe6 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -941,6 +994,7 @@ processors: target_field: _ingest._value.policy.id ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_7c5caaac field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -955,6 +1009,7 @@ processors: target_field: _ingest._value.requirement.description ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_352fb955 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -969,6 +1024,7 @@ processors: target_field: _ingest._value.requirement.id ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_92b02933 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -983,6 +1039,7 @@ processors: target_field: _ingest._value.requirement.name ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_f1ddc704 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -997,6 +1054,7 @@ processors: target_field: _ingest._value.section.description ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_79893455 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1011,6 +1069,7 @@ processors: target_field: _ingest._value.section.id ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_1bf8d450 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1025,6 +1084,7 @@ processors: target_field: _ingest._value.section.label ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_b6231f30 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1039,6 +1099,7 @@ processors: target_field: _ingest._value.standard.description ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_12ea5a97 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1053,6 +1114,7 @@ processors: target_field: _ingest._value.standard.id ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_48b54ab9 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1067,6 +1129,7 @@ processors: target_field: _ingest._value.standard.name ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_3b717880 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1081,6 +1144,7 @@ processors: ignore_missing: true tag: remove_prisma_cloud_alert_risk_detail_policy_scores_compliance_metadata_customAssigned - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_155d4995 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1091,6 +1155,7 @@ processors: target_field: _ingest._value.created.by ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_11f2f51f field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1105,6 +1170,7 @@ processors: field: _ingest._value.createdOn ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_ebf2bef1 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1122,6 +1188,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_332fa1c5 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1139,6 +1206,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_ddf9ada7 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1149,6 +1217,7 @@ processors: target_field: _ingest._value.finding_types ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_b0a1c2b1 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1159,6 +1228,7 @@ processors: target_field: _ingest._value.last_modified.by ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_2af72b23 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1173,6 +1243,7 @@ processors: field: _ingest._value.lastModifiedOn ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_272e9a49 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1190,6 +1261,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_36a2bd83 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1200,6 +1272,7 @@ processors: target_field: _ingest._value.policy.id ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_1a02fdc5 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1210,6 +1283,7 @@ processors: target_field: _ingest._value.policy.subtypes ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_3cc68442 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1220,6 +1294,7 @@ processors: target_field: _ingest._value.policy.type ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_38a894a5 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1230,6 +1305,7 @@ processors: target_field: _ingest._value.policy.upi ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_46849a26 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1247,6 +1323,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_061079b1 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1257,6 +1334,7 @@ processors: target_field: _ingest._value.remediation.cli_script_template ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_7186b610 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1275,6 +1353,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_6e26b9f9 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1293,6 +1372,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_ca96c7ae field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1311,6 +1391,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_78bd225d field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1321,6 +1402,7 @@ processors: target_field: _ingest._value.rule.api_name ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_e02c00ba field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1331,6 +1413,7 @@ processors: target_field: _ingest._value.rule.cloud.account ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_0a4228ab field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1341,6 +1424,7 @@ processors: target_field: _ingest._value.rule.cloud.type ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_6223f910 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1351,6 +1435,7 @@ processors: target_field: _ingest._value.rule.data_criteria ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_2b2aec78 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1361,6 +1446,7 @@ processors: target_field: _ingest._value.rule.data_criteria.classification_result ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_b17e97cf field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1371,6 +1457,7 @@ processors: target_field: _ingest._value.rule.resource.id_path ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_b2693638 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1381,6 +1468,7 @@ processors: target_field: _ingest._value.rule.resource.type ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_2915a80e field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1395,6 +1483,7 @@ processors: field: _ingest._value.ruleLastModifiedOn ignore_missing: true - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_59fa0eb6 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1413,6 +1502,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_alert_risk_detail_policy_scores_cfeeeeb5 field: prisma_cloud.alert.risk_detail.policy_scores if: ctx.prisma_cloud?.alert?.risk_detail?.policy_scores instanceof List ignore_failure: true @@ -1436,6 +1526,7 @@ processors: if: ctx.prisma_cloud?.alert?.risk_detail?.riskScore?.maxScore != '' on_failure: - append: + tag: append_error_message_0ffc731b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1447,11 +1538,12 @@ processors: if: ctx.prisma_cloud?.alert?.risk_detail?.riskScore?.score != '' on_failure: - append: + tag: append_error_message_e974f7c4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: field: json.saveSearchId - tag: rename_saveSearchId + tag: rename_json_saveSearchId_to_prisma_cloud_alert_save_search_id_a8e32b4d target_field: prisma_cloud.alert.save_search_id ignore_missing: true - rename: @@ -1506,6 +1598,7 @@ processors: description: Drops null/empty values recursively. on_failure: - append: + tag: append_error_message_7ab3f1a2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1514,6 +1607,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/prisma_cloud/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/prisma_cloud/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 97e69d0b4a8..8a3b5eebeea 100644 --- a/packages/prisma_cloud/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/prisma_cloud/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -31,11 +31,13 @@ processors: ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - drop: + tag: drop_41e92579 if: ctx.event?.original != null && ctx.event.original.isEmpty() - json: field: event.original @@ -43,9 +45,11 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_f3d9ca7e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - fingerprint: + tag: fingerprint_cce993bd fields: - json.action - json.timestamp @@ -60,6 +64,7 @@ processors: if: ctx.json?.ipAddress != '' && ctx.json?.ipAddress != 'RedLock Internal IP' on_failure: - append: + tag: append_error_message_3251dda5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -93,6 +98,7 @@ processors: if: ctx.user?.email != null on_failure: - append: + tag: append_error_message_5e1446ed field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -125,6 +131,7 @@ processors: if: ctx.prisma_cloud?.audit?.action?.type != '' on_failure: - append: + tag: append_error_message_480c0009 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag fail-{{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - split: @@ -134,6 +141,7 @@ processors: if: ctx.event?.action != null && ctx.event?.action != '' on_failure: - append: + tag: append_error_message_b5187eaa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - join: @@ -143,6 +151,7 @@ processors: if: ctx.event?.action instanceof List on_failure: - append: + tag: append_error_message_93eae4e8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -181,6 +190,7 @@ processors: } on_failure: - append: + tag: append_error_message_39b2060b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -191,20 +201,23 @@ processors: if: ctx.json?.timestamp != null && ctx.json.timestamp != '' on_failure: - append: + tag: append_error_message_724a2153 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag fail-{{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: field: json.timestamp - tag: 'date_rename_timestamp_to_custom_name' + tag: date_json_timestamp_to_prisma_cloud_audit_timestamp_289110dc target_field: prisma_cloud.audit.timestamp formats: - UNIX_MS if: ctx.json?.timestamp != null && ctx.json.timestamp != '' on_failure: - append: + tag: append_error_message_4243fa74 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag fail-{{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: + tag: remove_e5a80934 field: - json ignore_missing: true @@ -219,6 +232,7 @@ processors: - prisma_cloud.audit.result - prisma_cloud.audit.timestamp - script: + tag: script_06971239 lang: painless description: Drops null/empty values recursively. source: |- @@ -236,10 +250,12 @@ processors: } drop(ctx); - set: + tag: set_event_kind_92954dfa field: event.kind value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/prisma_cloud/data_stream/host/elasticsearch/ingest_pipeline/default.yml b/packages/prisma_cloud/data_stream/host/elasticsearch/ingest_pipeline/default.yml index a92937869b6..f308326d71a 100644 --- a/packages/prisma_cloud/data_stream/host/elasticsearch/ingest_pipeline/default.yml +++ b/packages/prisma_cloud/data_stream/host/elasticsearch/ingest_pipeline/default.yml @@ -35,11 +35,13 @@ processors: ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - drop: + tag: drop_41e92579 if: ctx.event?.original != null && ctx.event.original.isEmpty() - json: field: event.original @@ -47,6 +49,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_a6e0e953 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -68,6 +71,7 @@ processors: if: ctx.json?.agentless != '' on_failure: - append: + tag: append_error_message_27ec182e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -81,6 +85,7 @@ processors: target_field: prisma_cloud.host.all_compliance.data ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_all_compliance_data_f625a6e3 field: prisma_cloud.host.all_compliance.data if: ctx.prisma_cloud?.host?.all_compliance?.data instanceof List ignore_failure: true @@ -91,6 +96,7 @@ processors: value: '{{{_ingest._value.cve}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_all_compliance_data_9be42a81 field: prisma_cloud.host.all_compliance.data if: ctx.prisma_cloud?.host?.all_compliance?.data instanceof List ignore_failure: true @@ -122,9 +128,11 @@ processors: if: ctx.prisma_cloud?.host?.all_compliance?.enabled != '' on_failure: - remove: + tag: remove_prisma_cloud_host_all_compliance_enabled_ea65b8d4 field: prisma_cloud.host.all_compliance.enabled ignore_missing: true - append: + tag: append_error_message_9712a4b8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -136,6 +144,7 @@ processors: if: ctx.json?.appEmbedded != '' on_failure: - append: + tag: append_error_message_3cf4a1d3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -144,6 +153,7 @@ processors: target_field: prisma_cloud.host.applications ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_applications_9974f877 field: prisma_cloud.host.applications if: ctx.prisma_cloud?.host?.applications instanceof List ignore_failure: true @@ -162,6 +172,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_applications_340b9df2 field: prisma_cloud.host.applications if: ctx.prisma_cloud?.host?.applications instanceof List ignore_failure: true @@ -180,6 +191,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_applications_0f4785a0 field: prisma_cloud.host.applications if: ctx.prisma_cloud?.host?.applications instanceof List ignore_failure: true @@ -194,6 +206,7 @@ processors: field: _ingest._value.layerTime ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_applications_0017f337 field: prisma_cloud.host.applications if: ctx.prisma_cloud?.host?.applications instanceof List ignore_failure: true @@ -211,6 +224,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_applications_de4b8a9e field: prisma_cloud.host.applications if: ctx.prisma_cloud?.host?.applications instanceof List ignore_failure: true @@ -233,6 +247,7 @@ processors: target_field: prisma_cloud.host.binaries ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_binaries_b08b6015 field: prisma_cloud.host.binaries if: ctx.prisma_cloud?.host?.binaries instanceof List ignore_failure: true @@ -250,6 +265,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_binaries_4c46880e field: prisma_cloud.host.binaries if: ctx.prisma_cloud?.host?.binaries instanceof List ignore_failure: true @@ -268,6 +284,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_binaries_f8914cc8 field: prisma_cloud.host.binaries if: ctx.prisma_cloud?.host?.binaries instanceof List ignore_failure: true @@ -286,6 +303,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_binaries_3ab38422 field: prisma_cloud.host.binaries if: ctx.prisma_cloud?.host?.binaries instanceof List ignore_failure: true @@ -296,6 +314,7 @@ processors: target_field: _ingest._value.function_layer ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_binaries_5ae627be field: prisma_cloud.host.binaries if: ctx.prisma_cloud?.host?.binaries instanceof List ignore_failure: true @@ -314,6 +333,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_binaries_d6ca9ad3 field: prisma_cloud.host.binaries if: ctx.prisma_cloud?.host?.binaries instanceof List ignore_failure: true @@ -324,6 +344,7 @@ processors: target_field: _ingest._value.pkg_root_dir ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_binaries_621a7555 field: prisma_cloud.host.binaries if: ctx.prisma_cloud?.host?.binaries instanceof List ignore_failure: true @@ -361,6 +382,7 @@ processors: target_field: prisma_cloud.host.cloud_metadata.labels ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_cloud_metadata_labels_c8c15eb7 field: prisma_cloud.host.cloud_metadata.labels if: ctx.prisma_cloud?.host?.cloud_metadata?.labels instanceof List ignore_failure: true @@ -371,6 +393,7 @@ processors: target_field: _ingest._value.source.name ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_cloud_metadata_labels_76e34322 field: prisma_cloud.host.cloud_metadata.labels if: ctx.prisma_cloud?.host?.cloud_metadata?.labels instanceof List ignore_failure: true @@ -381,6 +404,7 @@ processors: target_field: _ingest._value.source.type ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_cloud_metadata_labels_8200e181 field: prisma_cloud.host.cloud_metadata.labels if: ctx.prisma_cloud?.host?.cloud_metadata?.labels instanceof List ignore_failure: true @@ -463,6 +487,7 @@ processors: if: ctx.json?.complianceDistribution?.critical != '' on_failure: - append: + tag: append_error_message_99509d33 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -474,6 +499,7 @@ processors: if: ctx.json?.complianceDistribution?.high != '' on_failure: - append: + tag: append_error_message_b26ebc0f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -485,6 +511,7 @@ processors: if: ctx.json?.complianceDistribution?.low != '' on_failure: - append: + tag: append_error_message_245b416d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -496,6 +523,7 @@ processors: if: ctx.json?.complianceDistribution?.medium != '' on_failure: - append: + tag: append_error_message_5c2d30d3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -507,6 +535,7 @@ processors: if: ctx.json?.complianceDistribution?.total != '' on_failure: - append: + tag: append_error_message_63604aa5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -515,6 +544,7 @@ processors: target_field: prisma_cloud.host.compliance_issues.data ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_compliance_issues_data_085ebd2b field: prisma_cloud.host.compliance_issues.data if: ctx.prisma_cloud?.host?.compliance_issues?.data instanceof List ignore_failure: true @@ -525,6 +555,7 @@ processors: value: '{{{_ingest._value.cve}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_compliance_issues_data_3bef114d field: prisma_cloud.host.compliance_issues.data if: ctx.prisma_cloud?.host?.compliance_issues?.data instanceof List ignore_failure: true @@ -543,6 +574,7 @@ processors: if: ctx.json?.complianceIssuesCount != '' on_failure: - append: + tag: append_error_message_5c83c7a6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -568,6 +600,7 @@ processors: if: ctx.json?.complianceRiskScore != '' on_failure: - append: + tag: append_error_message_cc9de48c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -579,6 +612,7 @@ processors: if: ctx.json?.creationTime != null && ctx.json.creationTime != '' on_failure: - append: + tag: append_error_message_7f8b0f84 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -607,6 +641,7 @@ processors: target_field: prisma_cloud.host.external_labels ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_external_labels_252722db field: prisma_cloud.host.external_labels if: ctx.prisma_cloud?.host?.external_labels instanceof List ignore_failure: true @@ -617,6 +652,7 @@ processors: target_field: _ingest._value.source.name ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_external_labels_620e1e26 field: prisma_cloud.host.external_labels if: ctx.prisma_cloud?.host?.external_labels instanceof List ignore_failure: true @@ -627,6 +663,7 @@ processors: target_field: _ingest._value.source.type ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_external_labels_70e1e014 field: prisma_cloud.host.external_labels if: ctx.prisma_cloud?.host?.external_labels instanceof List ignore_failure: true @@ -646,6 +683,7 @@ processors: target_field: prisma_cloud.host.files ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_files_c62cd889 field: prisma_cloud.host.files if: ctx.prisma_cloud?.host?.files instanceof List ignore_failure: true @@ -656,6 +694,7 @@ processors: value: '{{{_ingest._value.md5}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_files_4c86901c field: prisma_cloud.host.files if: ctx.prisma_cloud?.host?.files instanceof List ignore_failure: true @@ -666,6 +705,7 @@ processors: value: '{{{_ingest._value.md5}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_files_d3fade54 field: prisma_cloud.host.files if: ctx.prisma_cloud?.host?.files instanceof List ignore_failure: true @@ -676,6 +716,7 @@ processors: value: '{{{_ingest._value.path}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_files_490cffe5 field: prisma_cloud.host.files if: ctx.prisma_cloud?.host?.files instanceof List ignore_failure: true @@ -686,6 +727,7 @@ processors: value: '{{{_ingest._value.sha1}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_files_a17236f8 field: prisma_cloud.host.files if: ctx.prisma_cloud?.host?.files instanceof List ignore_failure: true @@ -696,6 +738,7 @@ processors: value: '{{{_ingest._value.sha1}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_files_71feeb64 field: prisma_cloud.host.files if: ctx.prisma_cloud?.host?.files instanceof List ignore_failure: true @@ -706,6 +749,7 @@ processors: value: '{{{_ingest._value.sha256}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_files_cf92b710 field: prisma_cloud.host.files if: ctx.prisma_cloud?.host?.files instanceof List ignore_failure: true @@ -716,16 +760,17 @@ processors: value: '{{{_ingest._value.sha256}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_files_a715c7d0 field: prisma_cloud.host.files if: ctx.prisma_cloud?.host?.files instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) ignore_failure: true processor: remove: field: - - _ingest._value.md5 - - _ingest._value.path - - _ingest._value.sha1 - - _ingest._value.sha256 + - _ingest._value.md5 + - _ingest._value.path + - _ingest._value.sha1 + - _ingest._value.sha256 ignore_missing: true tag: remove_file_hash_fields_ecs_duplicate - rename: @@ -741,9 +786,11 @@ processors: if: ctx.prisma_cloud?.host?.firewall_protection?.enabled != '' on_failure: - remove: + tag: remove_prisma_cloud_host_firewall_protection_enabled_c7f78ea4 field: prisma_cloud.host.firewall_protection.enabled ignore_missing: true - append: + tag: append_error_message_8bd8c5c9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -759,9 +806,11 @@ processors: if: ctx.prisma_cloud?.host?.firewall_protection?.ports != '' on_failure: - remove: + tag: remove_prisma_cloud_host_firewall_protection_ports_3e135d49 field: prisma_cloud.host.firewall_protection.ports ignore_missing: true - append: + tag: append_error_message_628e3123 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -772,9 +821,11 @@ processors: if: ctx.prisma_cloud?.host?.firewall_protection?.supported != '' on_failure: - remove: + tag: remove_prisma_cloud_host_firewall_protection_supported_99575fe3 field: prisma_cloud.host.firewall_protection.supported ignore_missing: true - append: + tag: append_error_message_db2e6acf field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -786,9 +837,11 @@ processors: if: ctx.prisma_cloud?.host?.firewall_protection?.tlsPorts != '' on_failure: - remove: + tag: remove_prisma_cloud_host_firewall_protection_tlsPorts_c29934e0 field: prisma_cloud.host.firewall_protection.tlsPorts ignore_missing: true - append: + tag: append_error_message_bb03c559 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -797,6 +850,7 @@ processors: target_field: prisma_cloud.host.firewall_protection.unprotected_processes ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_firewall_protection_unprotected_processes_7c00d7e3 field: prisma_cloud.host.firewall_protection.unprotected_processes if: ctx.prisma_cloud?.host?.firewall_protection?.unprotected_processes instanceof List ignore_failure: true @@ -814,6 +868,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_firewall_protection_unprotected_processes_8156d57e field: prisma_cloud.host.firewall_protection.unprotected_processes if: ctx.prisma_cloud?.host?.firewall_protection?.unprotected_processes instanceof List ignore_failure: true @@ -839,6 +894,7 @@ processors: if: ctx.json?.firstScanTime != null && ctx.json.firstScanTime != '' on_failure: - append: + tag: append_error_message_94b0ffc9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -847,6 +903,7 @@ processors: target_field: prisma_cloud.host.history ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_history_f55179fd field: prisma_cloud.host.history if: ctx.prisma_cloud?.host?.history instanceof List ignore_failure: true @@ -865,6 +922,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_history_79dfb037 field: prisma_cloud.host.history if: ctx.prisma_cloud?.host?.history instanceof List ignore_failure: true @@ -883,6 +941,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_history_a0169543 field: prisma_cloud.host.history if: ctx.prisma_cloud?.host?.history instanceof List ignore_failure: true @@ -897,6 +956,7 @@ processors: field: _ingest._value.created ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_history_da815e65 field: prisma_cloud.host.history if: ctx.prisma_cloud?.host?.history instanceof List ignore_failure: true @@ -915,6 +975,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_history_ed028f3f field: prisma_cloud.host.history if: ctx.prisma_cloud?.host?.history instanceof List ignore_failure: true @@ -927,6 +988,7 @@ processors: tag: remove_history_fields_duplicate ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_history_612a91b7 field: prisma_cloud.host.history if: ctx.prisma_cloud?.host?.history instanceof List ignore_failure: true @@ -941,6 +1003,7 @@ processors: value: '{{{_ingest._value.cve}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_history_6e54b5d3 field: prisma_cloud.host.history if: ctx.prisma_cloud?.host?.history instanceof List ignore_failure: true @@ -976,6 +1039,7 @@ processors: target_field: prisma_cloud.host.devices ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_devices_be010083 field: prisma_cloud.host.devices if: ctx.prisma_cloud?.host?.devices instanceof List ignore_failure: true @@ -993,6 +1057,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_devices_c467290a field: prisma_cloud.host.devices if: ctx.prisma_cloud?.host?.devices instanceof List ignore_failure: true @@ -1003,6 +1068,7 @@ processors: value: '{{{_ingest._value.ip}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_devices_20df14f2 field: prisma_cloud.host.devices if: ctx.prisma_cloud?.host?.devices instanceof List ignore_failure: true @@ -1013,6 +1079,7 @@ processors: value: '{{{_ingest._value.ip}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_devices_c467290a field: prisma_cloud.host.devices if: ctx.prisma_cloud?.host?.devices instanceof List ignore_failure: true @@ -1023,6 +1090,7 @@ processors: value: '{{{_ingest._value.ip}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_devices_5393c8b2 field: prisma_cloud.host.devices if: ctx.prisma_cloud?.host?.devices instanceof List ignore_failure: true @@ -1033,14 +1101,15 @@ processors: value: '{{{_ingest._value.name}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_devices_ec5a137c field: prisma_cloud.host.devices if: ctx.prisma_cloud?.host?.devices instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) ignore_failure: true processor: remove: field: - - _ingest._value.ip - - _ingest._value.name + - _ingest._value.ip + - _ingest._value.name ignore_missing: true tag: foreach_remove_ecs_duplicate_devices_fields - rename: @@ -1068,6 +1137,7 @@ processors: if: ctx.json?.hostRuntimeEnabled != '' on_failure: - append: + tag: append_error_message_144f16fb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1077,7 +1147,7 @@ processors: ignore_missing: true - rename: field: json.id - tag: rename_id + tag: rename_json_id_to_prisma_cloud_host_id_076caa5a target_field: prisma_cloud.host.id ignore_missing: true - date: @@ -1089,6 +1159,7 @@ processors: if: ctx.json?.image?.created != null && ctx.json.image.created != '' on_failure: - append: + tag: append_error_message_00858304 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1110,6 +1181,7 @@ processors: if: ctx.json?.image?.healthcheck != '' on_failure: - append: + tag: append_error_message_e5d49d7e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1118,6 +1190,7 @@ processors: target_field: prisma_cloud.host.image.history ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_image_history_8e041a90 field: prisma_cloud.host.image.history if: ctx.prisma_cloud?.host?.image?.history instanceof List ignore_failure: true @@ -1136,6 +1209,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_image_history_f635dcf4 field: prisma_cloud.host.image.history if: ctx.prisma_cloud?.host?.image?.history instanceof List ignore_failure: true @@ -1154,6 +1228,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_image_history_bf11b0be field: prisma_cloud.host.image.history if: ctx.prisma_cloud?.host?.image?.history instanceof List ignore_failure: true @@ -1168,6 +1243,7 @@ processors: field: _ingest._value.created ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_image_history_9df7820a field: prisma_cloud.host.image.history if: ctx.prisma_cloud?.host?.image?.history instanceof List ignore_failure: true @@ -1186,6 +1262,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_image_history_b7b3fe17 field: prisma_cloud.host.image.history if: ctx.prisma_cloud?.host?.image?.history instanceof List ignore_failure: true @@ -1198,6 +1275,7 @@ processors: tag: foreach_remove_image_history_duplicate_fields ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_image_history_626eb359 field: prisma_cloud.host.image.history if: ctx.prisma_cloud?.host?.image?.history instanceof List ignore_failure: true @@ -1212,6 +1290,7 @@ processors: value: '{{{_ingest._value.cve}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_image_history_a8162a14 field: prisma_cloud.host.image.history if: ctx.prisma_cloud?.host?.image?.history instanceof List ignore_failure: true @@ -1290,6 +1369,7 @@ processors: if: ctx.prisma_cloud?.host?.installed_products?.agentless != '' on_failure: - append: + tag: append_error_message_1c865dc2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1316,11 +1396,12 @@ processors: if: ctx.prisma_cloud?.host?.installed_products?.awsCloud != '' on_failure: - append: + tag: append_error_message_ad8dc5ec field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: field: json.installedProducts.clusterType - tag: rename_clusterType + tag: rename_json_installedProducts_clusterType_to_prisma_cloud_host_installed_products_cluster_type_b683e043 target_field: prisma_cloud.host.installed_products.cluster_type ignore_missing: true - convert: @@ -1332,6 +1413,7 @@ processors: if: ctx.prisma_cloud?.host?.installed_products?.crio != '' on_failure: - append: + tag: append_error_message_a515a84e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1343,6 +1425,7 @@ processors: if: ctx.prisma_cloud?.host?.installed_products?.dockerEnterprise != '' on_failure: - append: + tag: append_error_message_c90fdbb5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1354,6 +1437,7 @@ processors: if: ctx.prisma_cloud?.host?.installed_products?.hasPackageManager != '' on_failure: - append: + tag: append_error_message_170b220c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1365,6 +1449,7 @@ processors: if: ctx.prisma_cloud?.host?.installed_products?.k8sApiServer != '' on_failure: - append: + tag: append_error_message_3f427ee6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1376,6 +1461,7 @@ processors: if: ctx.prisma_cloud?.host?.installed_products?.k8sControllerManager != '' on_failure: - append: + tag: append_error_message_700721f8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1387,6 +1473,7 @@ processors: if: ctx.prisma_cloud?.host?.installed_products?.k8sEtcd != '' on_failure: - append: + tag: append_error_message_e81e262b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1398,6 +1485,7 @@ processors: if: ctx.prisma_cloud?.host?.installed_products?.k8sFederationApiServer != '' on_failure: - append: + tag: append_error_message_04737abb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1409,6 +1497,7 @@ processors: if: ctx.prisma_cloud?.host?.installed_products?.k8sFederationControllerManager != '' on_failure: - append: + tag: append_error_message_640f3689 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1420,6 +1509,7 @@ processors: if: ctx.prisma_cloud?.host?.installed_products?.k8sKubelet != '' on_failure: - append: + tag: append_error_message_0b0cacff field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1431,6 +1521,7 @@ processors: if: ctx.prisma_cloud?.host?.installed_products?.k8sProxy != '' on_failure: - append: + tag: append_error_message_a9c54953 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1442,6 +1533,7 @@ processors: if: ctx.prisma_cloud?.host?.installed_products?.k8sScheduler != '' on_failure: - append: + tag: append_error_message_313c973b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1458,6 +1550,7 @@ processors: if: ctx.prisma_cloud?.host?.installed_products?.openshift != '' on_failure: - append: + tag: append_error_message_5fbd2942 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1479,6 +1572,7 @@ processors: if: ctx.prisma_cloud?.host?.installed_products?.serverless != '' on_failure: - append: + tag: append_error_message_d974b384 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1490,6 +1584,7 @@ processors: if: ctx.prisma_cloud?.host?.installed_products?.swarmManager != '' on_failure: - append: + tag: append_error_message_26d0d2f6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1501,6 +1596,7 @@ processors: if: ctx.prisma_cloud?.host?.installed_products?.swarmNode != '' on_failure: - append: + tag: append_error_message_b83e1ef4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1509,6 +1605,7 @@ processors: target_field: prisma_cloud.host.instances ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_instances_d44981a5 field: prisma_cloud.host.instances if: ctx.prisma_cloud?.host?.instances instanceof List ignore_failure: true @@ -1531,6 +1628,7 @@ processors: if: ctx.json?.isARM64 != '' on_failure: - append: + tag: append_error_message_ae364c61 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1552,6 +1650,7 @@ processors: if: ctx.json?.malwareAnalyzedTime != null && ctx.json.malwareAnalyzedTime != '' on_failure: - append: + tag: append_error_message_38e72d97 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1563,6 +1662,7 @@ processors: if: ctx.json?.missingDistroVulnCoverage != '' on_failure: - append: + tag: append_error_message_7031ddb3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1572,7 +1672,7 @@ processors: ignore_missing: true - rename: field: json.osDistro - tag: rename_osDistro + tag: rename_json_osDistro_to_prisma_cloud_host_os_distro_value_a5b45a0e target_field: prisma_cloud.host.os_distro.value ignore_missing: true - set: @@ -1609,6 +1709,7 @@ processors: if: ctx.json?.packageCorrelationDone != '' on_failure: - append: + tag: append_error_message_1ad10c1b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1620,6 +1721,7 @@ processors: if: ctx.json?.packageManager != '' on_failure: - append: + tag: append_error_message_d37c0338 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1628,6 +1730,7 @@ processors: target_field: prisma_cloud.host.packages ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_packages_6547d6b8 field: prisma_cloud.host.packages if: ctx.prisma_cloud?.host?.packages instanceof List ignore_failure: true @@ -1638,6 +1741,7 @@ processors: target_field: _ingest._value.pkgs_type ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_packages_cff5f086 field: prisma_cloud.host.packages if: ctx.prisma_cloud?.host?.packages instanceof List ignore_failure: true @@ -1648,16 +1752,18 @@ processors: value: '{{{_ingest._value.pkgs_type}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_packages_a114b8c1 field: prisma_cloud.host.packages if: ctx.prisma_cloud?.host?.packages instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) ignore_failure: true processor: remove: field: - - _ingest._value.pkgs_type + - _ingest._value.pkgs_type ignore_missing: true tag: foreach_remove_packages_ecs_duplicate_fields - foreach: + tag: foreach_prisma_cloud_host_packages_bbe11bb5 field: prisma_cloud.host.packages if: ctx.prisma_cloud?.host?.packages instanceof List ignore_failure: true @@ -1680,6 +1786,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_packages_1587b8ac field: prisma_cloud.host.packages if: ctx.prisma_cloud?.host?.packages instanceof List ignore_failure: true @@ -1694,6 +1801,7 @@ processors: target_field: _ingest._value.binary_pkgs ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_packages_82f50bcb field: prisma_cloud.host.packages if: ctx.prisma_cloud?.host?.packages instanceof List ignore_failure: true @@ -1716,6 +1824,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_packages_e3ff87bf field: prisma_cloud.host.packages if: ctx.prisma_cloud?.host?.packages instanceof List ignore_failure: true @@ -1738,6 +1847,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_packages_c22a8a65 field: prisma_cloud.host.packages if: ctx.prisma_cloud?.host?.packages instanceof List ignore_failure: true @@ -1752,6 +1862,7 @@ processors: target_field: _ingest._value.function_layer ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_packages_3fbfbb89 field: prisma_cloud.host.packages if: ctx.prisma_cloud?.host?.packages instanceof List ignore_failure: true @@ -1774,6 +1885,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_packages_24a2020e field: prisma_cloud.host.packages if: ctx.prisma_cloud?.host?.packages instanceof List ignore_failure: true @@ -1788,6 +1900,7 @@ processors: target_field: _ingest._value.jar_identifier ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_packages_d96e37c1 field: prisma_cloud.host.packages if: ctx.prisma_cloud?.host?.packages instanceof List ignore_failure: true @@ -1806,6 +1919,7 @@ processors: field: _ingest._value.layerTime ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_packages_f6875b18 field: prisma_cloud.host.packages if: ctx.prisma_cloud?.host?.packages instanceof List ignore_failure: true @@ -1820,6 +1934,7 @@ processors: value: '{{{_ingest._value.license}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_packages_0172836d field: prisma_cloud.host.packages if: ctx.prisma_cloud?.host?.packages instanceof List ignore_failure: true @@ -1834,6 +1949,7 @@ processors: value: '{{{_ingest._value.name}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_packages_5974a2c4 field: prisma_cloud.host.packages if: ctx.prisma_cloud?.host?.packages instanceof List ignore_failure: true @@ -1848,6 +1964,7 @@ processors: value: '{{{_ingest._value.path}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_packages_6bcecf6b field: prisma_cloud.host.packages if: ctx.prisma_cloud?.host?.packages instanceof List ignore_failure: true @@ -1862,6 +1979,7 @@ processors: value: '{{{_ingest._value.version}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_packages_af927871 field: prisma_cloud.host.packages if: ctx.prisma_cloud?.host?.packages instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) ignore_failure: true @@ -1872,13 +1990,14 @@ processors: processor: remove: field: - - _ingest._value.license - - _ingest._value.name - - _ingest._value.path - - _ingest._value.version + - _ingest._value.license + - _ingest._value.name + - _ingest._value.path + - _ingest._value.version ignore_missing: true tag: remove_packages_ecs_duplicate_fields_custom - foreach: + tag: foreach_prisma_cloud_host_packages_47635b1d field: prisma_cloud.host.packages if: ctx.prisma_cloud?.host?.packages instanceof List ignore_failure: true @@ -1901,6 +2020,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_packages_e23e2952 field: prisma_cloud.host.packages if: ctx.prisma_cloud?.host?.packages instanceof List ignore_failure: true @@ -1928,6 +2048,7 @@ processors: if: ctx.json?.pullDuration != '' on_failure: - append: + tag: append_error_message_53217ef5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -1939,6 +2060,7 @@ processors: if: ctx.json?.pushTime != null && ctx.json.pushTime != '' on_failure: - append: + tag: append_error_message_7aef63bf field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1950,6 +2072,7 @@ processors: if: ctx.json?.redHatNonRPMImage != '' on_failure: - append: + tag: append_error_message_7ab137d2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -2017,6 +2140,7 @@ processors: if: ctx.json?.scanBuildDate != null && ctx.json.scanBuildDate != '' on_failure: - append: + tag: append_error_message_ce18e172 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2028,6 +2152,7 @@ processors: if: ctx.json?.scanDuration != '' on_failure: - append: + tag: append_error_message_bb2ec710 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -2044,6 +2169,7 @@ processors: if: ctx.json?.scanTime != null && ctx.json.scanTime != '' on_failure: - append: + tag: append_error_message_0de5a779 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -2080,6 +2206,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_startup_binaries_acd71f06 field: prisma_cloud.host.startup_binaries if: ctx.prisma_cloud?.host?.startup_binaries instanceof List ignore_failure: true @@ -2098,6 +2225,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_startup_binaries_f033bd40 field: prisma_cloud.host.startup_binaries if: ctx.prisma_cloud?.host?.startup_binaries instanceof List ignore_failure: true @@ -2116,6 +2244,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_startup_binaries_dd6e30b5 field: prisma_cloud.host.startup_binaries ignore_failure: true processor: @@ -2125,6 +2254,7 @@ processors: target_field: _ingest._value.function_layer ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_startup_binaries_75cd0d26 field: prisma_cloud.host.startup_binaries if: ctx.prisma_cloud?.host?.startup_binaries instanceof List ignore_failure: true @@ -2143,6 +2273,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_startup_binaries_2dcc13ec field: prisma_cloud.host.startup_binaries ignore_failure: true processor: @@ -2152,6 +2283,7 @@ processors: target_field: _ingest._value.pkg_root_dir ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_startup_binaries_8e63914e field: prisma_cloud.host.startup_binaries if: ctx.prisma_cloud?.host?.startup_binaries instanceof List ignore_failure: true @@ -2172,6 +2304,7 @@ processors: if: ctx.json?.stopped != '' on_failure: - append: + tag: append_error_message_01542d46 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -2195,6 +2328,7 @@ processors: target_field: prisma_cloud.host.trust_result.groups ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_trust_result_groups_8c5a1fa2 field: prisma_cloud.host.trust_result.groups if: ctx.prisma_cloud?.host?.trust_result?.groups instanceof List ignore_failure: true @@ -2205,6 +2339,7 @@ processors: value: '{{{_ingest._value._id}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_trust_result_groups_3f7ff0b3 field: prisma_cloud.host.trust_result.groups if: ctx.prisma_cloud?.host?.trust_result?.groups instanceof List ignore_failure: true @@ -2222,6 +2357,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_trust_result_groups_6a14b9e2 field: prisma_cloud.host.trust_result.groups if: ctx.prisma_cloud?.host?.trust_result?.groups instanceof List ignore_failure: true @@ -2236,6 +2372,7 @@ processors: field: _ingest._value.modified ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_trust_result_groups_4b6651b1 field: prisma_cloud.host.trust_result.groups if: ctx.prisma_cloud?.host?.trust_result?.groups instanceof List ignore_failure: true @@ -2246,6 +2383,7 @@ processors: value: '{{{_ingest._value.name}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_trust_result_groups_b9babc0f field: prisma_cloud.host.trust_result.groups if: ctx.prisma_cloud?.host?.trust_result?.groups instanceof List ignore_failure: true @@ -2256,18 +2394,20 @@ processors: value: '{{{_ingest._value.owner}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_trust_result_groups_2a5d5bd3 field: prisma_cloud.host.trust_result.groups if: ctx.prisma_cloud?.host?.trust_result?.groups instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) ignore_failure: true processor: remove: field: - - _ingest._value.id - - _ingest._value.name - - _ingest._value.owner + - _ingest._value.id + - _ingest._value.name + - _ingest._value.owner ignore_missing: true tag: foreach_remove_ecs_mapped_trustResult_fields - foreach: + tag: foreach_prisma_cloud_host_trust_result_groups_4af10e88 field: prisma_cloud.host.trust_result.groups if: ctx.prisma_cloud?.host?.trust_result?.groups instanceof List ignore_failure: true @@ -2291,6 +2431,7 @@ processors: if: ctx.json?.twistlockImage != '' on_failure: - append: + tag: append_error_message_8d3c1b1d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -2309,6 +2450,7 @@ processors: target_field: prisma_cloud.host.vulnerabilities.data ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_vulnerabilities_data_1d48868b field: prisma_cloud.host.vulnerabilities.data if: ctx.prisma_cloud?.host?.vulnerabilities?.data instanceof List ignore_failure: true @@ -2319,6 +2461,7 @@ processors: value: '{{{_ingest._value.cve}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_vulnerabilities_data_b05c7b97 field: prisma_cloud.host.vulnerabilities.data if: ctx.prisma_cloud?.host?.vulnerabilities?.data instanceof List ignore_failure: true @@ -2337,6 +2480,7 @@ processors: if: ctx.json?.vulnerabilitiesCount != '' on_failure: - append: + tag: append_error_message_44730caa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -2362,6 +2506,7 @@ processors: if: ctx.json?.vulnerabilityDistribution?.critical != '' on_failure: - append: + tag: append_error_message_f7e3a768 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2373,6 +2518,7 @@ processors: if: ctx.json?.vulnerabilityDistribution?.high != '' on_failure: - append: + tag: append_error_message_898c5168 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2384,6 +2530,7 @@ processors: if: ctx.json?.vulnerabilityDistribution?.low != '' on_failure: - append: + tag: append_error_message_570abe88 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2395,6 +2542,7 @@ processors: if: ctx.json?.vulnerabilityDistribution?.medium != '' on_failure: - append: + tag: append_error_message_ed4d0778 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2406,6 +2554,7 @@ processors: if: ctx.json?.vulnerabilityDistribution?.total != '' on_failure: - append: + tag: append_error_message_2d61ed94 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2417,6 +2566,7 @@ processors: if: ctx.json?.vulnerabilityRiskScore != '' on_failure: - append: + tag: append_error_message_770f9953 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2428,6 +2578,7 @@ processors: if: ctx.json?.wildFireUsage?.bytes != '' on_failure: - append: + tag: append_error_message_ba7be936 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2439,6 +2590,7 @@ processors: if: ctx.json?.wildFireUsage?.queries != '' on_failure: - append: + tag: append_error_message_81bab600 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2450,6 +2602,7 @@ processors: if: ctx.json?.wildFireUsage?.uploads != '' on_failure: - append: + tag: append_error_message_13926758 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' @@ -2510,6 +2663,7 @@ processors: tag: script_drop_null_values on_failure: - append: + tag: append_error_message_d707d903 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -2518,6 +2672,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/prisma_cloud/data_stream/host_profile/elasticsearch/ingest_pipeline/default.yml b/packages/prisma_cloud/data_stream/host_profile/elasticsearch/ingest_pipeline/default.yml index f72254c4375..de53c789db1 100644 --- a/packages/prisma_cloud/data_stream/host_profile/elasticsearch/ingest_pipeline/default.yml +++ b/packages/prisma_cloud/data_stream/host_profile/elasticsearch/ingest_pipeline/default.yml @@ -35,11 +35,13 @@ processors: ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - drop: + tag: drop_41e92579 if: ctx.event?.original != null && ctx.event.original.isEmpty() - json: field: event.original @@ -47,6 +49,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_f3d9ca7e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -89,6 +92,7 @@ processors: if: ctx.json?.created != null && ctx.json.created != '' on_failure: - append: + tag: append_error_message_400166d1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag fail-{{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -112,6 +116,7 @@ processors: target_field: prisma_cloud.host_profile.geoip.countries ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_profile_geoip_countries_a8866601 field: prisma_cloud.host_profile.geoip.countries if: ctx.prisma_cloud?.host_profile?.geoip?.countries instanceof List ignore_failure: true @@ -129,6 +134,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_profile_geoip_countries_1aa26fa8 field: prisma_cloud.host_profile.geoip.countries if: ctx.prisma_cloud?.host_profile?.geoip?.countries instanceof List ignore_failure: true @@ -139,6 +145,7 @@ processors: value: '{{{_ingest._value.ip}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_profile_geoip_countries_eda5fe94 field: prisma_cloud.host_profile.geoip.countries if: ctx.prisma_cloud?.host_profile?.geoip?.countries instanceof List ignore_failure: true @@ -161,6 +168,7 @@ processors: if: ctx.json?.geoip?.modified != null && ctx.json.geoip.modified != '' on_failure: - append: + tag: append_error_message_3bc412c7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag fail-{{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -172,6 +180,7 @@ processors: if: ctx.json?.hash != '' on_failure: - append: + tag: append_error_message_fa573b65 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -185,6 +194,7 @@ processors: target_field: prisma_cloud.host_profile.apps ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_profile_apps_1bf76f52 field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -195,6 +205,7 @@ processors: target_field: _ingest._value.startup_process ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_profile_apps_83709707 field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -205,6 +216,7 @@ processors: value: '{{{_ingest._value.startup_process.command}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_profile_apps_b4b324d9 field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -222,6 +234,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_profile_apps_c14be5d4 field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -232,6 +245,7 @@ processors: value: '{{{_ingest._value.startup_process.interactive}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_profile_apps_83f29a9e field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -242,6 +256,7 @@ processors: value: '{{{_ingest._value.startup_process.md5}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_profile_apps_e657852e field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -252,6 +267,7 @@ processors: value: '{{{_ingest._value.startup_process.md5}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_profile_apps_25da8a75 field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -269,6 +285,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_profile_apps_77f88030 field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -283,6 +300,7 @@ processors: field: _ingest._value.startup_process.time ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_profile_apps_4d906489 field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) ignore_failure: true @@ -295,6 +313,7 @@ processors: - _ingest._value.startup_process.md5 ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_profile_apps_c2a37650 field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -305,6 +324,7 @@ processors: target_field: _ingest._value.outgoing_ports ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_profile_apps_dbd50a71 field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -319,6 +339,7 @@ processors: target_field: _ingest._value.process_path ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_profile_apps_c1467fd8 field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -340,6 +361,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_profile_apps_88406eda field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -354,6 +376,7 @@ processors: value: '{{{_ingest._value.ip}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_profile_apps_9700a920 field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -375,6 +398,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_profile_apps_6f8f1ffa field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -393,6 +417,7 @@ processors: field: _ingest._value.modified ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_profile_apps_12a60059 field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -403,6 +428,7 @@ processors: target_field: _ingest._value.listening_ports ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_profile_apps_c0bedf10 field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -417,6 +443,7 @@ processors: target_field: _ingest._value.process_path ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_profile_apps_08944461 field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -435,6 +462,7 @@ processors: field: _ingest._value.modified ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_profile_apps_e3a4bcb5 field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -456,6 +484,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_profile_apps_1fdbe9f4 field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -477,6 +506,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_profile_apps_02b6ab8d field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -491,6 +521,7 @@ processors: tag: append_prisma_cloud_host_profile_apps_processes_interactive_into_process_interactive allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_profile_apps_825e0b12 field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -509,6 +540,7 @@ processors: field: _ingest._value.time ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_profile_apps_53f70fbf field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -530,6 +562,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_profile_apps_ce3362f3 field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -544,6 +577,7 @@ processors: value: '{{{_ingest._value.md5}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_profile_apps_133cfbf2 field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -558,6 +592,7 @@ processors: value: '{{{_ingest._value.md5}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_profile_apps_9d3fe78e field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List ignore_failure: true @@ -572,6 +607,7 @@ processors: value: '{{{_ingest._value.command}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_profile_apps_a8fe760b field: prisma_cloud.host_profile.apps if: ctx.prisma_cloud?.host_profile?.apps instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) ignore_failure: true @@ -593,6 +629,7 @@ processors: target_field: prisma_cloud.host_profile.ssh_events ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_profile_ssh_events_c87b0f7b field: prisma_cloud.host_profile.ssh_events if: ctx.prisma_cloud?.host_profile?.ssh_events instanceof List ignore_failure: true @@ -610,6 +647,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_profile_ssh_events_df6ce6e8 field: prisma_cloud.host_profile.ssh_events if: ctx.prisma_cloud?.host_profile?.ssh_events instanceof List ignore_failure: true @@ -620,6 +658,7 @@ processors: tag: append_prisma_cloud_host_profile_ssh_events_interactive_into_process_interactive allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_profile_ssh_events_cce2c918 field: prisma_cloud.host_profile.ssh_events if: ctx.prisma_cloud?.host_profile?.ssh_events instanceof List ignore_failure: true @@ -637,6 +676,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_host_profile_ssh_events_ccc3dd19 field: prisma_cloud.host_profile.ssh_events if: ctx.prisma_cloud?.host_profile?.ssh_events instanceof List ignore_failure: true @@ -647,6 +687,7 @@ processors: value: '{{{_ingest._value.md5}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_profile_ssh_events_3d6c8100 field: prisma_cloud.host_profile.ssh_events if: ctx.prisma_cloud?.host_profile?.ssh_events instanceof List ignore_failure: true @@ -657,6 +698,7 @@ processors: value: '{{{_ingest._value.md5}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_profile_ssh_events_e4ee9fd3 field: prisma_cloud.host_profile.ssh_events if: ctx.prisma_cloud?.host_profile?.ssh_events instanceof List ignore_failure: true @@ -667,6 +709,7 @@ processors: value: '{{{_ingest._value.command}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_host_profile_ssh_events_ee7ef98e field: prisma_cloud.host_profile.ssh_events if: ctx.prisma_cloud?.host_profile?.ssh_events instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) ignore_failure: true @@ -679,6 +722,7 @@ processors: - _ingest._value.md5 ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_profile_ssh_events_7da3f301 field: prisma_cloud.host_profile.ssh_events if: ctx.prisma_cloud?.host_profile?.ssh_events instanceof List ignore_failure: true @@ -693,6 +737,7 @@ processors: field: _ingest._value.loginTime ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_profile_ssh_events_1af314b4 field: prisma_cloud.host_profile.ssh_events if: ctx.prisma_cloud?.host_profile?.ssh_events instanceof List ignore_failure: true @@ -702,6 +747,7 @@ processors: - _ingest._value.loginTime ignore_missing: true - foreach: + tag: foreach_prisma_cloud_host_profile_ssh_events_7280a467 field: prisma_cloud.host_profile.ssh_events if: ctx.prisma_cloud?.host_profile?.ssh_events instanceof List ignore_failure: true @@ -724,9 +770,11 @@ processors: if: ctx.json?.time != null && ctx.json.time != '' on_failure: - append: + tag: append_error_message_57c6ca31 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag fail-{{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: + tag: remove_e5a80934 field: - json ignore_missing: true @@ -740,6 +788,7 @@ processors: - prisma_cloud.host_profile.created - prisma_cloud.host_profile.labels - script: + tag: script_06971239 lang: painless description: Drops null/empty values recursively. source: |- @@ -757,10 +806,12 @@ processors: } drop(ctx); - set: + tag: set_event_kind_92954dfa field: event.kind value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false @@ -776,5 +827,3 @@ on_failure: field: tags value: preserve_original_event allow_duplicates: false - - diff --git a/packages/prisma_cloud/data_stream/incident_audit/elasticsearch/ingest_pipeline/default.yml b/packages/prisma_cloud/data_stream/incident_audit/elasticsearch/ingest_pipeline/default.yml index 549d5c1b525..416e10b9455 100644 --- a/packages/prisma_cloud/data_stream/incident_audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/prisma_cloud/data_stream/incident_audit/elasticsearch/ingest_pipeline/default.yml @@ -27,6 +27,7 @@ processors: ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: ctx.event?.original != null @@ -37,6 +38,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_f3d9ca7e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -68,6 +70,7 @@ processors: if: ctx.json?.acknowledged != '' on_failure: - append: + tag: append_error_message_3361908a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -86,6 +89,7 @@ processors: target_field: prisma_cloud.incident_audit.data ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_7fefad6b field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -96,6 +100,7 @@ processors: target_field: _ingest._value.app.value ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_e55f5ce7 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -106,6 +111,7 @@ processors: target_field: _ingest._value.app.id ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_37d07c82 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -116,6 +122,7 @@ processors: target_field: _ingest._value.account_id ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_29f40038 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -126,6 +133,7 @@ processors: value: '{{{_ingest._value.account_id}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_incident_audit_data_26aaee48 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -136,6 +144,7 @@ processors: target_field: _ingest._value.attack.techniques ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_2de4dd63 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -150,6 +159,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_incident_audit_data_f85291a7 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -160,6 +170,7 @@ processors: target_field: _ingest._value.attack.type ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_44f5d9d7 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -170,6 +181,7 @@ processors: value: '{{{_ingest._value.attack.type}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_incident_audit_data_636281f3 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -180,6 +192,7 @@ processors: target_field: _ingest._value.container.value ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_739b5dbf field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -197,6 +210,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_incident_audit_data_864330a3 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -207,6 +221,7 @@ processors: target_field: _ingest._value.container.id ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_1d112bb5 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -227,6 +242,7 @@ processors: value: '{{{_ingest._value.container.name}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_incident_audit_data_38a8cb37 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -266,7 +282,7 @@ processors: - foreach: field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List - tag: foreach_append_data_fqdn + tag: foreach_prisma_cloud_incident_audit_data_adc070bf processor: append: field: related.hosts @@ -274,6 +290,7 @@ processors: value: '{{{_ingest._value.fqdn}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_incident_audit_data_4bd3678a field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -284,6 +301,7 @@ processors: target_field: _ingest._value.function.value ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_cfaaa2da field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -294,6 +312,7 @@ processors: target_field: _ingest._value.function.id ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_74ec790b field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -304,6 +323,7 @@ processors: value: '{{{_ingest._value.hostname}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_incident_audit_data_cc8c80e1 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -314,6 +334,7 @@ processors: target_field: _ingest._value.image.id ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_0320ab7b field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -324,6 +345,7 @@ processors: target_field: _ingest._value.image.name ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_8501441b field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -334,6 +356,7 @@ processors: value: '{{{_ingest._value.image.name}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_incident_audit_data_db2424c6 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -351,6 +374,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_incident_audit_data_57e93723 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -368,6 +392,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_incident_audit_data_ad480606 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -378,6 +403,7 @@ processors: value: '{{{_ingest._value.ip}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_incident_audit_data_3db71d55 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -395,6 +421,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_incident_audit_data_048e9945 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -412,6 +439,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_prisma_cloud_incident_audit_data_9736557a field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -422,6 +450,7 @@ processors: target_field: _ingest._value.process_path ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_9c7b9f78 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -432,6 +461,7 @@ processors: target_field: _ingest._value.profile_id ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_440fecf6 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -442,6 +472,7 @@ processors: value: '{{{_ingest._value.provider}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_incident_audit_data_103a65aa field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -452,6 +483,7 @@ processors: target_field: _ingest._value.raw_event ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_52546670 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -462,6 +494,7 @@ processors: target_field: _ingest._value.request_id ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_36e2bd5b field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -472,6 +505,7 @@ processors: target_field: _ingest._value.resource_id ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_f3a58617 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -482,6 +516,7 @@ processors: target_field: _ingest._value.rule_name ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_67138e1d field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -492,6 +527,7 @@ processors: value: '{{{_ingest._value.rule_name}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_incident_audit_data_a3d603ef field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -506,6 +542,7 @@ processors: field: _ingest._value.time ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_dd9cc1fa field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -516,6 +553,7 @@ processors: value: '{{{_ingest._value.user}}}' allow_duplicates: false - foreach: + tag: foreach_prisma_cloud_incident_audit_data_54dbedac field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -526,6 +564,7 @@ processors: target_field: _ingest._value.vm_id ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_49a33ad3 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List ignore_failure: true @@ -536,18 +575,19 @@ processors: target_field: _ingest._value.wild_fire_report_url ignore_missing: true - foreach: + tag: foreach_prisma_cloud_incident_audit_data_da13caa6 field: prisma_cloud.incident_audit.data if: ctx.prisma_cloud?.incident_audit?.data instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) ignore_failure: true processor: remove: field: - - _ingest._value.attack.type - - _ingest._value.attack.techniques - - _ingest._value.container.name - - _ingest._value.image.name - - _ingest._value.rule_name - - _ingest._value.provider + - _ingest._value.attack.type + - _ingest._value.attack.techniques + - _ingest._value.container.name + - _ingest._value.image.name + - _ingest._value.rule_name + - _ingest._value.provider - _ingest._value.fqdn - _ingest._value.account_id - _ingest._value.os @@ -722,6 +762,7 @@ processors: if: ctx.json?.serialNum != '' on_failure: - append: + tag: append_error_message_2cc4cfd7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -733,6 +774,7 @@ processors: if: ctx.json?.shouldCollect != '' on_failure: - append: + tag: append_error_message_0dc11c0b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -754,6 +796,7 @@ processors: if: ctx.json?.windows != '' on_failure: - append: + tag: append_error_message_97eea770 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -764,6 +807,7 @@ processors: if: ctx.json?.time != null && ctx.json.time != '' on_failure: - append: + tag: append_error_message_ed69cd43 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag fail-{{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -775,9 +819,11 @@ processors: if: ctx.json?.time != null && ctx.json.time != '' on_failure: - append: + tag: append_error_message_a251c3d3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag fail-{{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: + tag: remove_e5a80934 field: - json ignore_missing: true @@ -799,6 +845,7 @@ processors: - prisma_cloud.incident_audit.time - prisma_cloud.incident_audit.category - script: + tag: script_06971239 lang: painless description: Drops null/empty values recursively. source: |- @@ -816,10 +863,12 @@ processors: } drop(ctx); - set: + tag: set_event_kind_92954dfa field: event.kind value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false @@ -834,4 +883,4 @@ on_failure: - append: field: tags value: preserve_original_event - allow_duplicates: false \ No newline at end of file + allow_duplicates: false diff --git a/packages/prisma_cloud/data_stream/misconfiguration/elasticsearch/ingest_pipeline/default.yml b/packages/prisma_cloud/data_stream/misconfiguration/elasticsearch/ingest_pipeline/default.yml index 3996c1ec6b8..f4cc17696cc 100644 --- a/packages/prisma_cloud/data_stream/misconfiguration/elasticsearch/ingest_pipeline/default.yml +++ b/packages/prisma_cloud/data_stream/misconfiguration/elasticsearch/ingest_pipeline/default.yml @@ -23,6 +23,7 @@ processors: ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: ctx.event?.original != null @@ -33,6 +34,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_f3d9ca7e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -181,6 +183,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ffc907fd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -206,6 +209,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_36faf9cb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -216,6 +220,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_314f72a5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -246,9 +251,11 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_prisma_cloud_misconfiguration_scanned_policy_passed_8375f4d5 field: prisma_cloud.misconfiguration.scanned_policy.passed ignore_missing: true - append: + tag: append_error_message_fd48fbf9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -270,6 +277,7 @@ processors: if: ctx.result?.evaluation == null ignore_empty_value: true - script: + tag: script_9bce9b3b lang: painless if: ctx.prisma_cloud?.misconfiguration?.scanned_policy?.severity instanceof String && ctx.prisma_cloud.misconfiguration.scanned_policy.severity != '' source: |- @@ -369,6 +377,7 @@ processors: if: ctx.result?.evaluation == 'unknown' ignore_empty_value: true - remove: + tag: remove_e5a80934 field: - json ignore_missing: true @@ -386,6 +395,7 @@ processors: - prisma_cloud.misconfiguration.scanned_policy.id - prisma_cloud.misconfiguration.unified_asset_id - script: + tag: script_06971239 lang: painless description: Drops null/empty values recursively. source: |- @@ -403,10 +413,12 @@ processors: } drop(ctx); - set: + tag: set_event_kind_92954dfa field: event.kind value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/prisma_cloud/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml b/packages/prisma_cloud/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml index 9c801bdb00d..f75428b1fe7 100644 --- a/packages/prisma_cloud/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml +++ b/packages/prisma_cloud/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml @@ -23,6 +23,7 @@ processors: ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: ctx.event?.original != null @@ -33,6 +34,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_f3d9ca7e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -193,6 +195,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f0d5051c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -216,6 +219,7 @@ processors: if: ctx.json?.DISCOVERED_DATE != null && ctx.json.DISCOVERED_DATE != '' on_failure: - append: + tag: append_error_message_aa6c0be5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -242,6 +246,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_0f4c5b1f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -259,19 +264,23 @@ processors: if: ctx.json?.FIX_DATE != null && ctx.json.FIX_DATE != '' on_failure: - append: + tag: append_error_message_f814f5da field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set__temp_fixVersionIsArray_9dfb31ae field: _temp.fixVersionIsArray value: true if: ctx.json?.FIX_VERSION != null && ctx.json.FIX_VERSION.startsWith('[') && ctx.json.FIX_VERSION.endsWith(']') - gsub: + tag: gsub_json_FIX_VERSION_to_prisma_cloud_vulnerability_fix_version_f7ef30c0 field: json.FIX_VERSION pattern: '[\[\]]' replacement: '' target_field: prisma_cloud.vulnerability.fix_version if: ctx._temp?.fixVersionIsArray == true - split: + tag: split_prisma_cloud_vulnerability_fix_version_to_prisma_cloud_vulnerability_fix_version_29b97219 field: prisma_cloud.vulnerability.fix_version separator: ',\s*' target_field: prisma_cloud.vulnerability.fix_version @@ -288,16 +297,19 @@ processors: copy_from: prisma_cloud.vulnerability.fix_version ignore_empty_value: true - set: + tag: set__temp_installedVersionsIsArray_e3db7e88 field: _temp.installedVersionsIsArray value: true if: ctx.json?.INSTALLED_VERSIONS != null && ctx.json.INSTALLED_VERSIONS.startsWith('[') && ctx.json.INSTALLED_VERSIONS.endsWith(']') - gsub: + tag: gsub_json_INSTALLED_VERSIONS_to_prisma_cloud_vulnerability_installed_versions_79bd73e2 field: json.INSTALLED_VERSIONS pattern: '[\[\]]' replacement: '' target_field: prisma_cloud.vulnerability.installed_versions if: ctx._temp?.installedVersionsIsArray == true - split: + tag: split_prisma_cloud_vulnerability_installed_versions_to_prisma_cloud_vulnerability_installed_versions_f61471bb field: prisma_cloud.vulnerability.installed_versions separator: ',\s*' target_field: prisma_cloud.vulnerability.installed_versions @@ -332,6 +344,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_dec3881b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -364,6 +377,7 @@ processors: if: ctx.json?.PUBLISHED_DATE != null && ctx.json.PUBLISHED_DATE != '' on_failure: - append: + tag: append_error_message_652679c3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -412,6 +426,7 @@ processors: } on_failure: - append: + tag: append_error_message_d9174ca9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -441,6 +456,7 @@ processors: ignore_empty_value: true if: ctx.vulnerability?.id?.startsWith('CVE') == true - remove: + tag: remove_ef4e960b field: - json - _temp @@ -462,6 +478,7 @@ processors: - prisma_cloud.vulnerability.published_date - prisma_cloud.vulnerability.severity - script: + tag: script_06971239 lang: painless description: Drops null/empty values recursively. source: |- @@ -479,10 +496,12 @@ processors: } drop(ctx); - set: + tag: set_event_kind_92954dfa field: event.kind value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/prisma_cloud/manifest.yml b/packages/prisma_cloud/manifest.yml index c26a1ab32b7..251d19c6b23 100644 --- a/packages/prisma_cloud/manifest.yml +++ b/packages/prisma_cloud/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.2.3 name: prisma_cloud title: "Palo Alto Prisma Cloud" -version: "4.2.1" +version: "4.2.2" description: "Collect logs from Prisma Cloud with Elastic Agent." type: integration categories: diff --git a/packages/proofpoint_essentials/changelog.yml b/packages/proofpoint_essentials/changelog.yml index 9b84f9ed092..cb718ac0511 100644 --- a/packages/proofpoint_essentials/changelog.yml +++ b/packages/proofpoint_essentials/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.1.3" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.1.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/proofpoint_essentials/data_stream/threat/elasticsearch/ingest_pipeline/default.yml b/packages/proofpoint_essentials/data_stream/threat/elasticsearch/ingest_pipeline/default.yml index 2ea640aa889..d9e0d5eced1 100644 --- a/packages/proofpoint_essentials/data_stream/threat/elasticsearch/ingest_pipeline/default.yml +++ b/packages/proofpoint_essentials/data_stream/threat/elasticsearch/ingest_pipeline/default.yml @@ -98,6 +98,7 @@ processors: # convert values - fingerprint: + tag: fingerprint_02243b84 fields: - event.original target_field: _id @@ -110,8 +111,10 @@ processors: if: ctx.proofpoint_essentials?.threat?.click_ip != '' on_failure: - remove: + tag: remove_proofpoint_essentials_threat_click_ip_771e4fd0 field: proofpoint_essentials.threat.click_ip - append: + tag: append_error_message_145b459c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -123,8 +126,10 @@ processors: if: ctx.proofpoint_essentials?.threat?.click_time != null && ctx.proofpoint_essentials.threat.click_time != '' on_failure: - remove: + tag: remove_proofpoint_essentials_threat_click_time_75e2d792 field: proofpoint_essentials.threat.click_time - append: + tag: append_error_message_2b10ffde field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -134,8 +139,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_proofpoint_essentials_threat_impostor_score_aa1f2ede field: proofpoint_essentials.threat.impostor_score - append: + tag: append_error_message_76a19322 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -145,8 +152,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_proofpoint_essentials_threat_malware_score_7315aa0a field: proofpoint_essentials.threat.malware_score - append: + tag: append_error_message_ac07809a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -156,8 +165,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_proofpoint_essentials_threat_message_size_e28a3c12 field: proofpoint_essentials.threat.message_size - append: + tag: append_error_message_734357eb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -169,8 +180,10 @@ processors: if: ctx.proofpoint_essentials?.threat?.message_time != null && ctx.proofpoint_essentials.threat.message_time != '' on_failure: - remove: + tag: remove_proofpoint_essentials_threat_message_time_de78ffb7 field: proofpoint_essentials.threat.message_time - append: + tag: append_error_message_08601bc0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -180,8 +193,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_proofpoint_essentials_threat_phish_score_3f0c73b2 field: proofpoint_essentials.threat.phish_score - append: + tag: append_error_message_9e12d7e7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -192,8 +207,10 @@ processors: if: ctx.proofpoint_essentials?.threat?.sender_ip != '' on_failure: - remove: + tag: remove_proofpoint_essentials_threat_sender_ip_df4aace9 field: proofpoint_essentials.threat.sender_ip - append: + tag: append_error_message_eb343152 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -203,8 +220,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_proofpoint_essentials_threat_spam_score_ab6776ba field: proofpoint_essentials.threat.spam_score - append: + tag: append_error_message_cefc5252 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -216,11 +235,14 @@ processors: if: ctx.proofpoint_essentials?.threat?.threat_time != null && ctx.proofpoint_essentials.threat.threat_time != '' on_failure: - remove: + tag: remove_proofpoint_essentials_threat_threat_time_4def2472 field: proofpoint_essentials.threat.threat_time - append: + tag: append_error_message_1f27dcec field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_proofpoint_essentials_threat_threats_info_map_40090037 field: proofpoint_essentials.threat.threats_info_map if: ctx.proofpoint_essentials?.threat?.threats_info_map instanceof List processor: @@ -234,13 +256,14 @@ processors: - remove: field: _ingest._value.threat_time - gsub: + tag: gsub_proofpoint_essentials_threat_message_id_71e48d48 field: proofpoint_essentials.threat.message_id pattern: '<|>' replacement: '' ignore_missing: true - # populate ECS fields + # populate ECS fields - set: field: '@timestamp' tag: set_@timestamp_from_click_time @@ -277,6 +300,7 @@ processors: ctx['@timestamp'] = ts.toString(); on_failure: - append: + tag: append_error_message_a2465cea field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' @@ -328,7 +352,7 @@ processors: field: observer.product tag: set_observer_product value: Proofpoint Essentials - + # destination.* - set: field: destination.ip @@ -365,6 +389,7 @@ processors: copy_from: proofpoint_essentials.threat.message_id ignore_empty_value: true - foreach: + tag: foreach_proofpoint_essentials_threat_recipient_1f31e019 field: proofpoint_essentials.threat.recipient if: ctx.proofpoint_essentials?.threat?.recipient instanceof List processor: @@ -400,6 +425,7 @@ processors: copy_from: proofpoint_essentials.threat.xmailer ignore_empty_value: true - foreach: + tag: foreach_proofpoint_essentials_threat_to_addresses_b18f058c field: proofpoint_essentials.threat.to_addresses if: ctx.proofpoint_essentials?.threat?.to_addresses instanceof List processor: @@ -430,16 +456,17 @@ processors: } on_failure: - append: + tag: append_error_message_5320c283 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - + # rule.* - set: field: rule.name tag: set_rule_name_from_quarantine_rule copy_from: proofpoint_essentials.threat.quarantine_rule ignore_empty_value: true - + # source.* - set: field: source.ip @@ -451,7 +478,7 @@ processors: target_field: source.geo tag: geoip_source_ip ignore_missing: true - + # threat.* - set: field: threat.indicator.url.original @@ -521,6 +548,7 @@ processors: } on_failure: - append: + tag: append_error_message_3587259d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' @@ -529,7 +557,7 @@ processors: field: proofpoint_essentials.threat.user_agent tag: user_agent_threat_user_agent ignore_missing: true - + # related.* - append: field: related.user @@ -544,6 +572,7 @@ processors: allow_duplicates: false if: ctx.proofpoint_essentials?.threat?.sender != null - foreach: + tag: foreach_proofpoint_essentials_threat_message_parts_e5d6e9a4 field: proofpoint_essentials.threat.message_parts if: ctx.proofpoint_essentials?.threat?.message_parts instanceof List processor: @@ -553,6 +582,7 @@ processors: value: '{{{_ingest._value.md5}}}' allow_duplicates: false - foreach: + tag: foreach_proofpoint_essentials_threat_message_parts_aab2d1ee field: proofpoint_essentials.threat.message_parts if: ctx.proofpoint_essentials?.threat?.message_parts instanceof List processor: @@ -562,6 +592,7 @@ processors: value: '{{{_ingest._value.sha256}}}' allow_duplicates: false - foreach: + tag: foreach_proofpoint_essentials_threat_recipient_6a6da89a field: proofpoint_essentials.threat.recipient if: ctx.proofpoint_essentials?.threat?.recipient instanceof List processor: @@ -569,7 +600,7 @@ processors: field: related.user tag: append_recipient_to_related_user value: '{{{_ingest._value}}}' - allow_duplicates: false + allow_duplicates: false - append: field: related.user tag: append_reply_to_address_to_related_user @@ -583,6 +614,7 @@ processors: allow_duplicates: false if: ctx.proofpoint_essentials?.threat?.sender_ip != null - foreach: + tag: foreach_proofpoint_essentials_threat_to_addresses_c60fe339 field: proofpoint_essentials.threat.to_addresses if: ctx.proofpoint_essentials?.threat?.to_addresses instanceof List processor: @@ -655,6 +687,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/proofpoint_essentials/manifest.yml b/packages/proofpoint_essentials/manifest.yml index fc093e9dc8b..0f24dcbf4c3 100644 --- a/packages/proofpoint_essentials/manifest.yml +++ b/packages/proofpoint_essentials/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: proofpoint_essentials title: Proofpoint Essentials -version: 1.1.2 +version: 1.1.3 description: Collect logs from Proofpoint Essentials with Elastic Agent. type: integration categories: diff --git a/packages/proofpoint_itm/changelog.yml b/packages/proofpoint_itm/changelog.yml index a97f328dd1c..a8a51247956 100644 --- a/packages/proofpoint_itm/changelog.yml +++ b/packages/proofpoint_itm/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.1.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.1.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/proofpoint_itm/data_stream/report/elasticsearch/ingest_pipeline/default.yml b/packages/proofpoint_itm/data_stream/report/elasticsearch/ingest_pipeline/default.yml index d824f06de44..6a2aec5c7fe 100644 --- a/packages/proofpoint_itm/data_stream/report/elasticsearch/ingest_pipeline/default.yml +++ b/packages/proofpoint_itm/data_stream/report/elasticsearch/ingest_pipeline/default.yml @@ -39,6 +39,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -70,6 +71,7 @@ processors: if: ctx.json?._time != null && ctx.json._time != '' on_failure: - append: + tag: append_error_message_e51b40ce field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -88,6 +90,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_7c5d3c09 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -140,6 +143,7 @@ processors: if: ctx.json?.createdAt != null && ctx.json.createdAt != '' on_failure: - append: + tag: append_error_message_b642bc92 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -227,6 +231,7 @@ processors: if: ctx.json?.observedAt != null && ctx.json.observedAt != '' on_failure: - append: + tag: append_error_message_d4031c7e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -282,6 +287,7 @@ processors: if: ctx.json?.remoteAddress != '' on_failure: - append: + tag: append_error_message_c79408f2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -321,6 +327,7 @@ processors: if: ctx.json?.risingValue != null && ctx.json.risingValue != '' on_failure: - append: + tag: append_error_message_e04d260a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -360,7 +367,7 @@ processors: ignore_empty_value: true - set: field: rule.name - tag: set_rule_name_from_report_rule_name + tag: set_rule_name_0d792bbc copy_from: proofpoint_itm.report.rule.name ignore_empty_value: true - rename: @@ -429,6 +436,7 @@ processors: if: ctx.json?.timezoneOffset != '' on_failure: - append: + tag: append_error_message_0869cc83 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -446,6 +454,7 @@ processors: if: ctx.json?.userActivityObservedAt != null && ctx.json.userActivityObservedAt != '' on_failure: - append: + tag: append_error_message_4dc786af field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -501,6 +510,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/proofpoint_itm/manifest.yml b/packages/proofpoint_itm/manifest.yml index ec06eb47060..35c0d4ea483 100644 --- a/packages/proofpoint_itm/manifest.yml +++ b/packages/proofpoint_itm/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: proofpoint_itm title: Proofpoint ITM -version: "1.1.1" +version: "1.1.2" description: Collect logs from Proofpoint ITM using Elastic Agent. type: integration categories: diff --git a/packages/proofpoint_on_demand/changelog.yml b/packages/proofpoint_on_demand/changelog.yml index 3d0af06f0d4..536e5ce93c6 100644 --- a/packages/proofpoint_on_demand/changelog.yml +++ b/packages/proofpoint_on_demand/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.10.3" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.10.2" changes: - description: Fix kibana.version constraint to use tilde ranges for intermediate version bounds. diff --git a/packages/proofpoint_on_demand/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/proofpoint_on_demand/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index c5e6d3dec5c..37369a5ca00 100644 --- a/packages/proofpoint_on_demand/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/proofpoint_on_demand/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -24,6 +24,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - fingerprint: @@ -212,6 +213,7 @@ processors: target_field: proofpoint_on_demand.audit.metadata.origin.type ignore_missing: true - foreach: + tag: foreach_json_metadata_trace_e1061e5c field: json.metadata.trace if: ctx.json?.metadata?.trace instanceof List processor: @@ -269,6 +271,7 @@ processors: if: ctx.json?.audit?.service?.ipAddress != '' on_failure: - append: + tag: append_error_message_d971b1b8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -278,6 +281,7 @@ processors: allow_duplicates: false if: ctx.proofpoint_on_demand?.audit?.service?.ip_address != null - foreach: + tag: foreach_json_audit_tags_b9759b49 field: json.audit.tags if: ctx.json?.audit?.tags instanceof List processor: @@ -338,6 +342,7 @@ processors: if: ctx.json?.ts != null && ctx.json.ts != '' on_failure: - append: + tag: append_error_message_60059ca1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -386,6 +391,7 @@ processors: if: ctx.json?.audit?.user?.ipAddress != '' on_failure: - append: + tag: append_error_message_75f29e56 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -466,6 +472,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/proofpoint_on_demand/data_stream/mail/elasticsearch/ingest_pipeline/default.yml b/packages/proofpoint_on_demand/data_stream/mail/elasticsearch/ingest_pipeline/default.yml index 9fae6bd547d..dfbe60aa0e2 100644 --- a/packages/proofpoint_on_demand/data_stream/mail/elasticsearch/ingest_pipeline/default.yml +++ b/packages/proofpoint_on_demand/data_stream/mail/elasticsearch/ingest_pipeline/default.yml @@ -24,6 +24,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - fingerprint: @@ -166,6 +167,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_cb31a21c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -260,6 +262,7 @@ processors: } on_failure: - append: + tag: append_error_message_5e37d66c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -280,6 +283,7 @@ processors: copy_from: proofpoint_on_demand.mail.sm.msgid ignore_empty_value: true - foreach: + tag: foreach_proofpoint_on_demand_mail_sm_to_a7466dee field: proofpoint_on_demand.mail.sm.to if: ctx.proofpoint_on_demand?.mail?.sm?.to instanceof List processor: @@ -289,6 +293,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - foreach: + tag: foreach_proofpoint_on_demand_mail_sm_to_7fc1540d field: proofpoint_on_demand.mail.sm.to if: ctx.proofpoint_on_demand?.mail?.sm?.to instanceof List processor: @@ -305,6 +310,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_001a9e22 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -315,6 +321,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f73ff24e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -349,6 +356,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_6226c45f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -405,6 +413,7 @@ processors: if: ctx.json?.ts != null && ctx.json.ts != '' on_failure: - append: + tag: append_error_message_fcb529a3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -455,6 +464,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/proofpoint_on_demand/data_stream/message/elasticsearch/ingest_pipeline/default.yml b/packages/proofpoint_on_demand/data_stream/message/elasticsearch/ingest_pipeline/default.yml index baa183e8a4f..b24f4f7be7e 100644 --- a/packages/proofpoint_on_demand/data_stream/message/elasticsearch/ingest_pipeline/default.yml +++ b/packages/proofpoint_on_demand/data_stream/message/elasticsearch/ingest_pipeline/default.yml @@ -24,6 +24,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - fingerprint: @@ -132,6 +133,7 @@ processors: if: ctx.json?.connection?.ip != '' on_failure: - append: + tag: append_error_message_ffa1d6d6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -189,6 +191,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4e5b9d9a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -232,6 +235,7 @@ processors: target_field: proofpoint_on_demand.message.envelope.from_hashed ignore_missing: true - foreach: + tag: foreach_json_envelope_rcpts_ad228241 field: json.envelope.rcpts if: ctx.json?.envelope?.rcpts instanceof List processor: @@ -251,6 +255,7 @@ processors: target_field: proofpoint_on_demand.message.envelope.rcpts_hashed ignore_missing: true - foreach: + tag: foreach_json_filter_actions_da0faab2 field: json.filter.actions if: ctx.json?.filter?.actions instanceof List processor: @@ -268,6 +273,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_filter_actions_869eb3d6 field: json.filter.actions if: ctx.json?.filter?.actions instanceof List processor: @@ -299,6 +305,7 @@ processors: if: ctx.json?.filter?.startTime != null && ctx.json.filter.startTime != '' on_failure: - append: + tag: append_error_message_b2665f4a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -314,6 +321,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ae454a24 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -329,6 +337,7 @@ processors: } on_failure: - append: + tag: append_error_message_af4e7b6d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -339,6 +348,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2ca8326f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -349,6 +359,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_c68767d8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -367,6 +378,7 @@ processors: target_field: proofpoint_on_demand.message.filter.modules.av.virus_names ignore_missing: true - foreach: + tag: foreach_json_filter_modules_dkimv_c2185086 field: json.filter.modules.dkimv if: ctx.json?.filter?.modules?.dkimv instanceof List processor: @@ -381,6 +393,7 @@ processors: target_field: proofpoint_on_demand.message.filter.modules.dkimv ignore_missing: true - foreach: + tag: foreach_json_filter_modules_dmarc_alignment_f56a6eb7 field: json.filter.modules.dmarc.alignment if: ctx.json?.filter?.modules?.dmarc?.alignment instanceof List processor: @@ -390,6 +403,7 @@ processors: target_field: _ingest._value.from_domain ignore_missing: true - foreach: + tag: foreach_json_filter_modules_dmarc_alignment_9add4e41 field: json.filter.modules.dmarc.alignment if: ctx.json?.filter?.modules?.dmarc?.alignment instanceof List processor: @@ -399,6 +413,7 @@ processors: value: '{{{_ingest._value.from_domain}}}' allow_duplicates: false - foreach: + tag: foreach_json_filter_modules_dmarc_alignment_de06bf97 field: json.filter.modules.dmarc.alignment if: ctx.json?.filter?.modules?.dmarc?.alignment instanceof List processor: @@ -416,6 +431,7 @@ processors: target_field: proofpoint_on_demand.message.filter.modules.dmarc.alignment ignore_missing: true - foreach: + tag: foreach_json_filter_modules_dmarc_authResults_bbc3eeeb field: json.filter.modules.dmarc.authResults if: ctx.json?.filter?.modules?.dmarc?.authResults instanceof List processor: @@ -425,6 +441,7 @@ processors: path: _ingest._value.emailIdentities ignore_failure: true - foreach: + tag: foreach_json_filter_modules_dmarc_authResults_be2797e4 field: json.filter.modules.dmarc.authResults if: ctx.json?.filter?.modules?.dmarc?.authResults instanceof List processor: @@ -434,6 +451,7 @@ processors: target_field: _ingest._value.email_identities ignore_missing: true - foreach: + tag: foreach_json_filter_modules_dmarc_authResults_aed085d3 field: json.filter.modules.dmarc.authResults if: ctx.json?.filter?.modules?.dmarc?.authResults instanceof List processor: @@ -443,6 +461,7 @@ processors: target_field: _ingest._value.email_identities.smtp.mailfrom_hashed ignore_missing: true - foreach: + tag: foreach_json_filter_modules_dmarc_authResults_4b203b2c field: json.filter.modules.dmarc.authResults if: ctx.json?.filter?.modules?.dmarc?.authResults instanceof List processor: @@ -452,6 +471,7 @@ processors: value: '{{{_ingest._value.email_identities.header.from}}}' allow_duplicates: false - foreach: + tag: foreach_json_filter_modules_dmarc_authResults_8051eb1f field: json.filter.modules.dmarc.authResults if: ctx.json?.filter?.modules?.dmarc?.authResults instanceof List processor: @@ -488,6 +508,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4e05f28d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -498,6 +519,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_929891ee field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -508,6 +530,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_067561ec field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -523,6 +546,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_28d0159e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -564,6 +588,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_319c1ba9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -574,6 +599,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2b065470 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -584,6 +610,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9fd9e770 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -594,6 +621,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_64b3ed69 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -604,6 +632,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_fad9bdc8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -614,6 +643,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f8fa8b56 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -624,6 +654,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_bbc5d492 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -634,6 +665,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_eab0d7ed field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -644,6 +676,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_ce4fe0f9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -654,6 +687,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_355f0137 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -664,6 +698,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_61293ea8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -691,6 +726,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_af418a89 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -781,6 +817,7 @@ processors: if: ctx.json?.filter?.throttleIp != '' on_failure: - append: + tag: append_error_message_fffdff0d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -790,6 +827,7 @@ processors: allow_duplicates: false if: ctx.proofpoint_on_demand?.message?.filter?.throttle_ip != null - foreach: + tag: foreach_json_filter_verified_rcpts_a1eb2943 field: json.filter.verified.rcpts if: ctx.json?.filter?.verified?.rcpts instanceof List processor: @@ -924,6 +962,7 @@ processors: override: true ignore_empty_value: true - foreach: + tag: foreach_json_msg_header_cc_fcc9adee field: json.msg.header.cc if: ctx.json?.msg?.header?.cc instanceof List processor: @@ -933,6 +972,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - foreach: + tag: foreach_json_msg_header_cc_505baafb field: json.msg.header.cc if: ctx.json?.msg?.header?.cc instanceof List processor: @@ -947,6 +987,7 @@ processors: target_field: proofpoint_on_demand.message.msg.header.cc ignore_missing: true - foreach: + tag: foreach_json_msg_header_from_ce990724 field: json.msg.header.from if: ctx.json?.msg?.header?.from instanceof List processor: @@ -956,6 +997,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - foreach: + tag: foreach_json_msg_header_from_732a05b5 field: json.msg.header.from if: ctx.json?.msg?.header?.from instanceof List processor: @@ -980,6 +1022,7 @@ processors: target_field: proofpoint_on_demand.message.msg.header.message_id ignore_missing: true - foreach: + tag: foreach_proofpoint_on_demand_message_msg_header_message_id_e736fa11 field: proofpoint_on_demand.message.msg.header.message_id if: ctx.proofpoint_on_demand?.message?.msg?.header?.message_id instanceof List processor: @@ -994,6 +1037,7 @@ processors: target_field: proofpoint_on_demand.message.msg.header.reply_to ignore_missing: true - foreach: + tag: foreach_proofpoint_on_demand_message_msg_header_reply_to_0d006883 field: proofpoint_on_demand.message.msg.header.reply_to if: ctx.proofpoint_on_demand?.message?.msg?.header?.reply_to instanceof List processor: @@ -1003,6 +1047,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - foreach: + tag: foreach_proofpoint_on_demand_message_msg_header_reply_to_48f1c4ea field: proofpoint_on_demand.message.msg.header.reply_to if: ctx.proofpoint_on_demand?.message?.msg?.header?.reply_to instanceof List processor: @@ -1017,6 +1062,7 @@ processors: target_field: proofpoint_on_demand.message.msg.header.return_path ignore_missing: true - foreach: + tag: foreach_json_msg_header_subject_120d9c67 field: json.msg.header.subject if: ctx.json?.msg?.header?.subject instanceof List processor: @@ -1031,6 +1077,7 @@ processors: target_field: proofpoint_on_demand.message.msg.header.subject ignore_missing: true - foreach: + tag: foreach_json_msg_header_to_290a8045 field: json.msg.header.to if: ctx.json?.msg?.header?.to instanceof List processor: @@ -1040,6 +1087,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - foreach: + tag: foreach_json_msg_header_to_426bf0ca field: json.msg.header.to if: ctx.json?.msg?.header?.to instanceof List processor: @@ -1094,6 +1142,7 @@ processors: target_field: proofpoint_on_demand.message.msg.normalized_header ignore_missing: true - foreach: + tag: foreach_json_msg_parsedAddresses_cc_1da6fd65 field: json.msg.parsedAddresses.cc if: ctx.json?.msg?.parsedAddresses?.cc instanceof List processor: @@ -1103,6 +1152,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - foreach: + tag: foreach_json_msg_parsedAddresses_from_f8f706af field: json.msg.parsedAddresses.from if: ctx.json?.msg?.parsedAddresses?.from instanceof List processor: @@ -1117,6 +1167,7 @@ processors: target_field: json.msg.parsedAddresses.from_hashed ignore_missing: true - foreach: + tag: foreach_json_msg_parsedAddresses_to_e964c83a field: json.msg.parsedAddresses.to if: ctx.json?.msg?.parsedAddresses?.to instanceof List processor: @@ -1143,11 +1194,12 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_08f577b6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: lang: painless - description: Map msgParts fields. + description: Map msgParts fields. if: ctx.json?.msgParts instanceof List params: 'dataBase64': 'database64' @@ -1278,9 +1330,11 @@ processors: } on_failure: - append: + tag: append_error_message_c5dec0a4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_email_attachments_7f15a33e field: email.attachments if: ctx.email?.attachments instanceof List processor: @@ -1297,6 +1351,7 @@ processors: if: ctx.json?.ts != null && ctx.json.ts != '' on_failure: - append: + tag: append_error_message_db669455 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1305,6 +1360,7 @@ processors: copy_from: proofpoint_on_demand.message.ts ignore_empty_value: true - foreach: + tag: foreach_proofpoint_on_demand_message_msg_parts_d7fae032 field: proofpoint_on_demand.message.msg_parts if: ctx.proofpoint_on_demand?.message?.msg_parts instanceof List processor: @@ -1317,6 +1373,7 @@ processors: ignore_missing: true if: ctx.tags == null || !ctx.tags.contains('preserve_duplicate_custom_fields') - foreach: + tag: foreach_proofpoint_on_demand_message_msg_parts_b5798d82 field: proofpoint_on_demand.message.msg_parts if: ctx.proofpoint_on_demand?.message?.msg_parts instanceof List processor: @@ -1381,6 +1438,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/proofpoint_on_demand/manifest.yml b/packages/proofpoint_on_demand/manifest.yml index 6d267673904..f8069ef94df 100644 --- a/packages/proofpoint_on_demand/manifest.yml +++ b/packages/proofpoint_on_demand/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.1.4 name: proofpoint_on_demand title: Proofpoint On Demand -version: "1.10.2" +version: "1.10.3" description: Collect logs from Proofpoint On Demand with Elastic Agent. type: integration categories: diff --git a/packages/proofpoint_tap/changelog.yml b/packages/proofpoint_tap/changelog.yml index 9e04d7de603..02b66916ca3 100644 --- a/packages/proofpoint_tap/changelog.yml +++ b/packages/proofpoint_tap/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.31.3" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.31.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/proofpoint_tap/data_stream/clicks_blocked/elasticsearch/ingest_pipeline/default.yml b/packages/proofpoint_tap/data_stream/clicks_blocked/elasticsearch/ingest_pipeline/default.yml index 69a5dce4234..c004d94c993 100644 --- a/packages/proofpoint_tap/data_stream/clicks_blocked/elasticsearch/ingest_pipeline/default.yml +++ b/packages/proofpoint_tap/data_stream/clicks_blocked/elasticsearch/ingest_pipeline/default.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Proofpoint TAP blocked clicks logs. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - remove: @@ -16,62 +17,77 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_cac66847 field: event.original target_field: json ignore_failure: true - drop: + tag: drop_91913a4c if: ctx.json?.clicksBlocked instanceof List && ctx.json.clicksBlocked.length == 0 - fingerprint: + tag: fingerprint_02243b84 fields: - event.original target_field: _id ignore_missing: true - append: + tag: append_event_category_df23064f field: event.category value: email ignore_failure: true - append: + tag: append_event_action_dce318aa field: event.action value: denied ignore_failure: true - append: + tag: append_event_type_61ba5509 field: event.type value: info ignore_failure: true - set: + tag: set_event_kind_de80643c field: event.kind value: event - rename: + tag: rename_json_sender_to_email_from_address_aefd139d field: json.sender target_field: email.from.address ignore_missing: true - set: + tag: set_email_from_address_fff7c0a5 field: email.from.address value: ['{{{email.from.address}}}'] if: ctx.email?.from?.address instanceof String - rename: + tag: rename_json_messageID_to_email_message_id_478a43db field: json.messageID target_field: email.message_id ignore_missing: true - rename: + tag: rename_json_recipient_to_email_to_address_d9c203ce field: json.recipient target_field: email.to.address ignore_missing: true - set: + tag: set_email_to_address_52631338 field: email.to.address value: ['{{{email.to.address}}}'] if: ctx.email?.to?.address instanceof String - date: + tag: date_json_clickTime_to_proofpoint_tap_clicks_blocked_click_time_e86b2632 field: json.clickTime target_field: proofpoint_tap.clicks_blocked.click_time if: ctx.json?.clickTime != null && ctx.json.clickTime != '' @@ -79,37 +95,46 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_b6a344fd field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_timestamp_0d4bb266 field: '@timestamp' copy_from: proofpoint_tap.clicks_blocked.click_time ignore_failure: true ignore_empty_value: true - rename: + tag: rename_json_id_to_event_id_6438e920 field: json.id target_field: event.id ignore_missing: true - convert: + tag: convert_json_clickIP_to_destination_ip_c503d1a2 field: json.clickIP target_field: destination.ip type: ip ignore_missing: true on_failure: - remove: + tag: remove_json_clickIP_b414ea43 field: json.clickIP - append: + tag: append_error_message_b66feaad field: error.message value: '{{{_ingest.on_failure_message}}}' - geoip: + tag: geoip_destination_ip_to_destination_geo_b3425e9d field: destination.ip target_field: destination.geo ignore_missing: true on_failure: - append: + tag: append_error_message_340fa3f4 field: error.message value: '{{{_ingest.on_failure_message}}}' - geoip: + tag: geoip_destination_ip_to_destination_as_ffe24f3a database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -119,28 +144,35 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d6b244b7 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - convert: + tag: convert_json_senderIP_to_source_ip_0e7c10d3 field: json.senderIP target_field: source.ip type: ip ignore_missing: true on_failure: - remove: + tag: remove_json_senderIP_d3b08891 field: json.senderIP - append: + tag: append_error_message_18a6114e field: error.message value: '{{{_ingest.on_failure_message}}}' - date: + tag: date_json_threatTime_to_proofpoint_tap_clicks_blocked_threat_time_38f872cf field: json.threatTime target_field: proofpoint_tap.clicks_blocked.threat.time if: ctx.json?.threatTime != null && ctx.json.threatTime != '' @@ -148,6 +180,7 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_7b09bb28 field: error.message value: '{{{_ingest.on_failure_message}}}' - script: @@ -163,61 +196,75 @@ processors: ctx['@timestamp'] = ctx.proofpoint_tap.clicks_blocked.threat.time; } - uri_parts: + tag: uri_parts_json_url_be09671e field: json.url keep_original: false on_failure: - append: + tag: append_error_message_ff176615 field: error.message value: '{{{_ingest.on_failure_message}}}' - urldecode: + tag: urldecode_json_url_to_url_full_28871040 field: json.url target_field: url.full ignore_missing: true - user_agent: + tag: user_agent_json_userAgent_to_user_agent_46c412e7 field: json.userAgent target_field: user_agent ignore_missing: true ignore_failure: true - rename: + tag: rename_json_campaignId_to_proofpoint_tap_clicks_blocked_campaign_id_2cd0864f field: json.campaignId target_field: proofpoint_tap.clicks_blocked.campaign_id ignore_missing: true - rename: + tag: rename_json_classification_to_proofpoint_tap_clicks_blocked_classification_ffa48ae6 field: json.classification target_field: proofpoint_tap.clicks_blocked.classification ignore_missing: true - rename: + tag: rename_json_GUID_to_proofpoint_tap_guid_ab57fad4 field: json.GUID target_field: proofpoint_tap.guid ignore_missing: true - rename: + tag: rename_json_threatID_to_proofpoint_tap_clicks_blocked_threat_id_4aef1f9a field: json.threatID target_field: proofpoint_tap.clicks_blocked.threat.id ignore_missing: true - rename: + tag: rename_json_threatStatus_to_proofpoint_tap_clicks_blocked_threat_status_e90f26d8 field: json.threatStatus target_field: proofpoint_tap.clicks_blocked.threat.status ignore_missing: true - rename: + tag: rename_json_threatURL_to_proofpoint_tap_clicks_blocked_threat_url_4e3c4dba field: json.threatURL target_field: proofpoint_tap.clicks_blocked.threat.url ignore_missing: true - append: + tag: append_related_ip_86c0561d field: related.ip value: '{{{source.ip}}}' if: ctx.source?.ip != null && ctx.source.ip != '' allow_duplicates: false ignore_failure: true - append: + tag: append_related_ip_e09ec9b6 field: related.ip value: '{{{destination.ip}}}' if: ctx.destination?.ip != null && ctx.destination.ip != '' allow_duplicates: false ignore_failure: true - remove: + tag: remove_json_29cdffdc field: json ignore_missing: true - script: + tag: script_258616c7 description: Drops null/empty values recursively. lang: painless source: | diff --git a/packages/proofpoint_tap/data_stream/clicks_permitted/elasticsearch/ingest_pipeline/default.yml b/packages/proofpoint_tap/data_stream/clicks_permitted/elasticsearch/ingest_pipeline/default.yml index ad752a7d4c6..76e98c49755 100644 --- a/packages/proofpoint_tap/data_stream/clicks_permitted/elasticsearch/ingest_pipeline/default.yml +++ b/packages/proofpoint_tap/data_stream/clicks_permitted/elasticsearch/ingest_pipeline/default.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Proofpoint TAP permitted clicks logs. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - remove: @@ -16,62 +17,77 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_cac66847 field: event.original target_field: json ignore_failure: true - drop: + tag: drop_3468b8b4 if: ctx.json?.clicksPermitted instanceof List && ctx.json.clicksPermitted.length == 0 - fingerprint: + tag: fingerprint_02243b84 fields: - event.original target_field: _id ignore_missing: true - append: + tag: append_event_category_df23064f field: event.category value: email ignore_failure: true - append: + tag: append_event_action_466170d5 field: event.action value: allowed ignore_failure: true - append: + tag: append_event_type_61ba5509 field: event.type value: info ignore_failure: true - set: + tag: set_event_kind_de80643c field: event.kind value: event - rename: + tag: rename_json_sender_to_email_from_address_aefd139d field: json.sender target_field: email.from.address ignore_missing: true - set: + tag: set_email_from_address_fff7c0a5 field: email.from.address value: ['{{{email.from.address}}}'] if: ctx.email?.from?.address instanceof String - rename: + tag: rename_json_messageID_to_email_message_id_478a43db field: json.messageID target_field: email.message_id ignore_missing: true - rename: + tag: rename_json_recipient_to_email_to_address_d9c203ce field: json.recipient target_field: email.to.address ignore_missing: true - set: + tag: set_email_to_address_52631338 field: email.to.address value: ['{{{email.to.address}}}'] if: ctx.email?.to?.address instanceof String - date: + tag: date_json_clickTime_to_proofpoint_tap_clicks_permitted_click_time_1ee48696 field: json.clickTime target_field: proofpoint_tap.clicks_permitted.click_time if: ctx.json?.clickTime != null && ctx.json.clickTime != '' @@ -79,37 +95,46 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_adbf1b21 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_timestamp_c37daf0e field: '@timestamp' copy_from: proofpoint_tap.clicks_permitted.click_time ignore_failure: true ignore_empty_value: true - rename: + tag: rename_json_id_to_event_id_6438e920 field: json.id target_field: event.id ignore_missing: true - convert: + tag: convert_json_clickIP_to_destination_ip_c503d1a2 field: json.clickIP target_field: destination.ip type: ip ignore_missing: true on_failure: - remove: + tag: remove_json_clickIP_b414ea43 field: json.clickIP - append: + tag: append_error_message_b66feaad field: error.message value: '{{{_ingest.on_failure_message}}}' - geoip: + tag: geoip_destination_ip_to_destination_geo_b3425e9d field: destination.ip target_field: destination.geo ignore_missing: true on_failure: - append: + tag: append_error_message_340fa3f4 field: error.message value: '{{{_ingest.on_failure_message}}}' - geoip: + tag: geoip_destination_ip_to_destination_as_ffe24f3a database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -119,28 +144,35 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d6b244b7 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - convert: + tag: convert_json_senderIP_to_source_ip_0e7c10d3 field: json.senderIP target_field: source.ip type: ip ignore_missing: true on_failure: - remove: + tag: remove_json_senderIP_d3b08891 field: json.senderIP - append: + tag: append_error_message_18a6114e field: error.message value: '{{{_ingest.on_failure_message}}}' - date: + tag: date_json_threatTime_to_proofpoint_tap_clicks_permitted_threat_time_f989427b field: json.threatTime target_field: proofpoint_tap.clicks_permitted.threat.time if: ctx.json?.threatTime != null && ctx.json.threatTime != '' @@ -148,6 +180,7 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_3a78e690 field: error.message value: '{{{_ingest.on_failure_message}}}' - script: @@ -163,61 +196,75 @@ processors: ctx['@timestamp'] = ctx.proofpoint_tap.clicks_permitted.threat.time; } - uri_parts: + tag: uri_parts_json_url_be09671e field: json.url keep_original: false on_failure: - append: + tag: append_error_message_ff176615 field: error.message value: '{{{_ingest.on_failure_message}}}' - urldecode: + tag: urldecode_json_url_to_url_full_28871040 field: json.url target_field: url.full ignore_missing: true - user_agent: + tag: user_agent_json_userAgent_to_user_agent_46c412e7 field: json.userAgent target_field: user_agent ignore_missing: true ignore_failure: true - rename: + tag: rename_json_campaignId_to_proofpoint_tap_clicks_permitted_campaign_id_2a8d6ad7 field: json.campaignId target_field: proofpoint_tap.clicks_permitted.campaign_id ignore_missing: true - rename: + tag: rename_json_classification_to_proofpoint_tap_clicks_permitted_classification_397d7cba field: json.classification target_field: proofpoint_tap.clicks_permitted.classification ignore_missing: true - rename: + tag: rename_json_GUID_to_proofpoint_tap_guid_ab57fad4 field: json.GUID target_field: proofpoint_tap.guid ignore_missing: true - rename: + tag: rename_json_threatID_to_proofpoint_tap_clicks_permitted_threat_id_8e50f5d6 field: json.threatID target_field: proofpoint_tap.clicks_permitted.threat.id ignore_missing: true - rename: + tag: rename_json_threatStatus_to_proofpoint_tap_clicks_permitted_threat_status_43c0eb28 field: json.threatStatus target_field: proofpoint_tap.clicks_permitted.threat.status ignore_missing: true - rename: + tag: rename_json_threatURL_to_proofpoint_tap_clicks_permitted_threat_url_4fe24982 field: json.threatURL target_field: proofpoint_tap.clicks_permitted.threat.url ignore_missing: true - append: + tag: append_related_ip_86c0561d field: related.ip value: '{{{source.ip}}}' if: ctx.source?.ip != null && ctx.source.ip != '' allow_duplicates: false ignore_failure: true - append: + tag: append_related_ip_e09ec9b6 field: related.ip value: '{{{destination.ip}}}' if: ctx.destination?.ip != null && ctx.destination.ip != '' allow_duplicates: false ignore_failure: true - remove: + tag: remove_json_29cdffdc field: json ignore_missing: true - script: + tag: script_258616c7 description: Drops null/empty values recursively. lang: painless source: | diff --git a/packages/proofpoint_tap/data_stream/message_blocked/elasticsearch/ingest_pipeline/default.yml b/packages/proofpoint_tap/data_stream/message_blocked/elasticsearch/ingest_pipeline/default.yml index c10035b6bc5..5ffb400edb9 100644 --- a/packages/proofpoint_tap/data_stream/message_blocked/elasticsearch/ingest_pipeline/default.yml +++ b/packages/proofpoint_tap/data_stream/message_blocked/elasticsearch/ingest_pipeline/default.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Proofpoint TAP blocked message logs. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - remove: @@ -16,77 +17,95 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_cac66847 field: event.original target_field: json ignore_failure: true - drop: + tag: drop_95015dc4 if: ctx.json?.messagesBlocked instanceof List && ctx.json.messagesBlocked.length == 0 - date: + tag: date_json_messageTime_a1f037bb field: json.messageTime if: ctx.json?.messageTime != null && ctx.json.messageTime != '' formats: - ISO8601 on_failure: - append: + tag: append_error_message_63547788 field: error.message value: '{{{_ingest.on_failure_message}}}' - fingerprint: + tag: fingerprint_02243b84 fields: - event.original target_field: _id ignore_missing: true - append: + tag: append_event_category_df23064f field: event.category value: email ignore_failure: true - append: + tag: append_event_action_dce318aa field: event.action value: denied ignore_failure: true - append: + tag: append_event_type_61ba5509 field: event.type value: info ignore_failure: true - set: + tag: set_event_kind_de80643c field: event.kind value: event - convert: + tag: convert_json_senderIP_to_source_ip_0e7c10d3 field: json.senderIP target_field: source.ip type: ip ignore_missing: true on_failure: - remove: + tag: remove_json_senderIP_d3b08891 field: json.senderIP - append: + tag: append_error_message_18a6114e field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_related_ip_86c0561d field: related.ip value: '{{{source.ip}}}' if: ctx.source?.ip != null && ctx.source.ip != '' allow_duplicates: false ignore_failure: true - geoip: + tag: geoip_source_ip_to_source_geo_5d3ae3d6 field: source.ip target_field: source.geo ignore_missing: true if: ctx.source?.ip != null && ctx.source.ip != '' on_failure: - append: + tag: append_error_message_900765d9 field: error.message value: '{{{_ingest.on_failure_message}}}' - geoip: + tag: geoip_source_ip_to_source_as_82271077 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -97,17 +116,21 @@ processors: if: ctx.source?.ip != null && ctx.source.ip != '' on_failure: - append: + tag: append_error_message_e390fa84 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - foreach: + tag: foreach_json_messageParts_a01b3372 field: json.messageParts processor: append: @@ -118,6 +141,7 @@ processors: ignore_failure: true if: ctx.json?.messageParts instanceof List - foreach: + tag: foreach_json_messageParts_255f7bfb field: json.messageParts processor: append: @@ -128,55 +152,68 @@ processors: ignore_failure: true if: ctx.json?.messageParts instanceof List - rename: + tag: rename_json_ccAddresses_to_email_cc_address_11b02c18 field: json.ccAddresses target_field: email.cc.address ignore_missing: true - set: + tag: set_email_cc_address_76afae5b field: email.cc.address value: ['{{{email.cc.address}}}'] if: ctx.email?.cc?.address instanceof String - set: + tag: set_email_delivery_timestamp_8dc48ca9 field: email.delivery_timestamp copy_from: '@timestamp' ignore_failure: true - rename: + tag: rename_json_fromAddress_to_email_from_address_54a1d768 field: json.fromAddress target_field: email.from.address ignore_missing: true - set: + tag: set_email_from_address_fff7c0a5 field: email.from.address value: ['{{{email.from.address}}}'] if: ctx.email?.from?.address instanceof String - rename: + tag: rename_json_messageID_to_email_message_id_478a43db field: json.messageID target_field: email.message_id ignore_missing: true - gsub: + tag: gsub_email_message_id_9c7cf5ea field: email.message_id pattern: '<|>' replacement: '' ignore_missing: true - rename: + tag: rename_json_replyToAddress_to_email_reply_to_address_f55651c7 field: json.replyToAddress target_field: email.reply_to.address ignore_missing: true - rename: + tag: rename_json_sender_to_email_sender_address_90c69416 field: json.sender target_field: email.sender.address ignore_missing: true - rename: + tag: rename_json_subject_to_email_subject_dcbc667a field: json.subject target_field: email.subject ignore_missing: true - set: + tag: set_email_to_address_00a46272 field: email.to.address copy_from: json.toAddresses ignore_failure: true - set: + tag: set_email_to_address_52631338 field: email.to.address value: ['{{{email.to.address}}}'] if: ctx.email?.to?.address instanceof String - foreach: + tag: foreach_json_recipient_c1c02972 field: json.recipient processor: append: @@ -187,18 +224,22 @@ processors: ignore_failure: true if: ctx.json?.recipient instanceof List - rename: + tag: rename_json_xmailer_to_email_x_mailer_7644d631 field: json.xmailer target_field: email.x_mailer ignore_missing: true - rename: + tag: rename_json_id_to_event_id_6438e920 field: json.id target_field: event.id ignore_missing: true - set: + tag: set_email_attachments_75c4fd67 field: email.attachments copy_from: json.messageParts ignore_failure: true - foreach: + tag: foreach_email_attachments_f22ed6f5 field: email.attachments processor: remove: @@ -210,6 +251,7 @@ processors: ignore_failure: true if: ctx.email?.attachments instanceof List - foreach: + tag: foreach_email_attachments_5e0aa7a8 field: email.attachments processor: rename: @@ -219,6 +261,7 @@ processors: ignore_failure: true if: ctx.email?.attachments instanceof List - foreach: + tag: foreach_email_attachments_88d690e8 field: email.attachments processor: rename: @@ -228,6 +271,7 @@ processors: ignore_failure: true if: ctx.email?.attachments instanceof List - foreach: + tag: foreach_email_attachments_1ec76ffe field: email.attachments processor: rename: @@ -237,6 +281,7 @@ processors: ignore_failure: true if: ctx.email?.attachments instanceof List - foreach: + tag: foreach_email_attachments_7d6b12e8 field: email.attachments processor: rename: @@ -246,6 +291,7 @@ processors: ignore_failure: true if: ctx.email?.attachments instanceof List - script: + tag: script_458cf990 description: Adding hash in related.hash from artifact field. lang: painless ignore_failure: true @@ -269,75 +315,93 @@ processors: } } - rename: + tag: rename_json_toAddresses_to_proofpoint_tap_message_blocked_to_addresses_9494e7e3 field: json.toAddresses target_field: proofpoint_tap.message_blocked.to_addresses ignore_missing: true - rename: + tag: rename_json_recipient_to_proofpoint_tap_message_blocked_recipient_3d84084e field: json.recipient target_field: proofpoint_tap.message_blocked.recipient ignore_missing: true - rename: + tag: rename_json_cluster_to_proofpoint_tap_message_blocked_cluster_10b147ae field: json.cluster target_field: proofpoint_tap.message_blocked.cluster ignore_missing: true - convert: + tag: convert_json_completelyRewritten_to_proofpoint_tap_message_blocked_completely_rewritten_4dc32c74 field: json.completelyRewritten target_field: proofpoint_tap.message_blocked.completely_rewritten type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_GUID_to_proofpoint_tap_guid_ab57fad4 field: json.GUID target_field: proofpoint_tap.guid ignore_missing: true - rename: + tag: rename_json_headerCC_to_proofpoint_tap_message_blocked_header_cc_797bb978 field: json.headerCC target_field: proofpoint_tap.message_blocked.header.cc ignore_missing: true - rename: + tag: rename_json_headerFrom_to_proofpoint_tap_message_blocked_header_from_4f6bea24 field: json.headerFrom target_field: proofpoint_tap.message_blocked.header.from ignore_missing: true - gsub: + tag: gsub_proofpoint_tap_message_blocked_header_from_443aa9de field: proofpoint_tap.message_blocked.header.from pattern: '<|>' replacement: '' ignore_missing: true - rename: + tag: rename_json_headerReplyTo_to_proofpoint_tap_message_blocked_header_replyto_56777162 field: json.headerReplyTo target_field: proofpoint_tap.message_blocked.header.replyto ignore_missing: true - rename: + tag: rename_json_headerTo_to_proofpoint_tap_message_blocked_header_to_b7e0f5a0 field: json.headerTo target_field: proofpoint_tap.message_blocked.header.to ignore_missing: true - convert: + tag: convert_json_impostorScore_to_proofpoint_tap_message_blocked_impostor_score_3dbd870d field: json.impostorScore target_field: proofpoint_tap.message_blocked.impostor_score type: double ignore_missing: true on_failure: - remove: + tag: remove_json_impostorScore_c3862a0a field: json.impostorScore - append: + tag: append_error_message_123d53e4 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_malwareScore_to_proofpoint_tap_message_blocked_malware_score_7f7b2b18 field: json.malwareScore target_field: proofpoint_tap.message_blocked.malware_score type: long ignore_missing: true on_failure: - remove: + tag: remove_json_malwareScore_de2cd0f9 field: json.malwareScore - append: + tag: append_error_message_1006c5a7 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_messageParts_to_proofpoint_tap_message_blocked_message_parts_77d87247 field: json.messageParts target_field: proofpoint_tap.message_blocked.message_parts ignore_missing: true - foreach: + tag: foreach_proofpoint_tap_message_blocked_message_parts_0f10002a field: proofpoint_tap.message_blocked.message_parts processor: remove: @@ -350,6 +414,7 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_blocked?.message_parts instanceof List - foreach: + tag: foreach_proofpoint_tap_message_blocked_message_parts_ad423c27 field: proofpoint_tap.message_blocked.message_parts processor: rename: @@ -359,6 +424,7 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_blocked?.message_parts instanceof List - foreach: + tag: foreach_proofpoint_tap_message_blocked_message_parts_f8880ba4 field: proofpoint_tap.message_blocked.message_parts processor: rename: @@ -368,63 +434,79 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_blocked?.message_parts instanceof List - convert: + tag: convert_json_messageSize_to_proofpoint_tap_message_blocked_message_size_04c39137 field: json.messageSize target_field: proofpoint_tap.message_blocked.message_size type: long ignore_missing: true on_failure: - remove: + tag: remove_json_messageSize_d6216bd1 field: json.messageSize - append: + tag: append_error_message_3dba3e5e field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_modulesRun_to_proofpoint_tap_message_blocked_modules_run_30024ec3 field: json.modulesRun target_field: proofpoint_tap.message_blocked.modules_run ignore_missing: true - convert: + tag: convert_json_phishScore_to_proofpoint_tap_message_blocked_phish_score_5b8d7383 field: json.phishScore target_field: proofpoint_tap.message_blocked.phish_score type: long ignore_missing: true on_failure: - remove: + tag: remove_json_phishScore_7cd80597 field: json.phishScore - append: + tag: append_error_message_76a9e01c field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_policyRoutes_to_proofpoint_tap_message_blocked_policy_routes_b8864ce3 field: json.policyRoutes target_field: proofpoint_tap.message_blocked.policy_routes ignore_missing: true - rename: + tag: rename_json_QID_to_proofpoint_tap_message_blocked_qid_1d1a55a2 field: json.QID target_field: proofpoint_tap.message_blocked.qid ignore_missing: true - rename: + tag: rename_json_quarantineFolder_to_proofpoint_tap_message_blocked_quarantine_folder_f9edb48a field: json.quarantineFolder target_field: proofpoint_tap.message_blocked.quarantine.folder ignore_missing: true - rename: + tag: rename_json_quarantineRule_to_proofpoint_tap_message_blocked_quarantine_rule_eeff680a field: json.quarantineRule target_field: proofpoint_tap.message_blocked.quarantine.rule ignore_missing: true - convert: + tag: convert_json_spamScore_to_proofpoint_tap_message_blocked_spam_score_3a339cd8 field: json.spamScore target_field: proofpoint_tap.message_blocked.spam_score type: long ignore_missing: true on_failure: - remove: + tag: remove_json_spamScore_f6b7964b field: json.spamScore - append: + tag: append_error_message_95041c61 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_threatsInfoMap_to_proofpoint_tap_message_blocked_threat_info_map_68cca403 field: json.threatsInfoMap target_field: proofpoint_tap.message_blocked.threat_info_map ignore_missing: true - foreach: + tag: foreach_proofpoint_tap_message_blocked_threat_info_map_315e367c field: proofpoint_tap.message_blocked.threat_info_map processor: rename: @@ -434,6 +516,7 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_blocked?.threat_info_map instanceof List - foreach: + tag: foreach_proofpoint_tap_message_blocked_threat_info_map_116656bb field: proofpoint_tap.message_blocked.threat_info_map processor: rename: @@ -443,6 +526,7 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_blocked?.threat_info_map instanceof List - foreach: + tag: foreach_proofpoint_tap_message_blocked_threat_info_map_53833de9 field: proofpoint_tap.message_blocked.threat_info_map processor: rename: @@ -452,6 +536,7 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_blocked?.threat_info_map instanceof List - foreach: + tag: foreach_proofpoint_tap_message_blocked_threat_info_map_650e1809 field: proofpoint_tap.message_blocked.threat_info_map processor: rename: @@ -461,6 +546,7 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_blocked?.threat_info_map instanceof List - foreach: + tag: foreach_proofpoint_tap_message_blocked_threat_info_map_01f49681 field: proofpoint_tap.message_blocked.threat_info_map processor: rename: @@ -470,6 +556,7 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_blocked?.threat_info_map instanceof List - foreach: + tag: foreach_proofpoint_tap_message_blocked_threat_info_map_f4c9f2c5 field: proofpoint_tap.message_blocked.threat_info_map processor: date: @@ -481,6 +568,7 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_blocked?.threat_info_map instanceof List - foreach: + tag: foreach_proofpoint_tap_message_blocked_threat_info_map_bdb3d2dd field: proofpoint_tap.message_blocked.threat_info_map processor: remove: @@ -489,6 +577,7 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_blocked?.threat_info_map instanceof List - foreach: + tag: foreach_proofpoint_tap_message_blocked_threat_info_map_f8eaca85 field: proofpoint_tap.message_blocked.threat_info_map processor: rename: @@ -498,6 +587,7 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_blocked?.threat_info_map instanceof List - foreach: + tag: foreach_proofpoint_tap_message_blocked_threat_info_map_662b5883 field: proofpoint_tap.message_blocked.threat_info_map processor: rename: @@ -520,10 +610,12 @@ processors: } } - remove: + tag: remove_e5a80934 field: - json ignore_missing: true - script: + tag: script_e2ccbdd7 description: Drops null/empty values recursively. lang: painless source: | diff --git a/packages/proofpoint_tap/data_stream/message_delivered/elasticsearch/ingest_pipeline/default.yml b/packages/proofpoint_tap/data_stream/message_delivered/elasticsearch/ingest_pipeline/default.yml index 2431fbd39d7..5e3d1d47225 100644 --- a/packages/proofpoint_tap/data_stream/message_delivered/elasticsearch/ingest_pipeline/default.yml +++ b/packages/proofpoint_tap/data_stream/message_delivered/elasticsearch/ingest_pipeline/default.yml @@ -2,6 +2,7 @@ description: Pipeline for parsing Proofpoint TAP delivered message logs. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - remove: @@ -16,73 +17,90 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_json_cac66847 field: event.original target_field: json ignore_failure: true - drop: + tag: drop_af5ef778 if: ctx.json?.messagesDelivered instanceof List && ctx.json.messagesDelivered.length == 0 - date: + tag: date_json_messageTime_a1f037bb field: json.messageTime if: ctx.json?.messageTime != null && ctx.json.messageTime != '' formats: - ISO8601 on_failure: - append: + tag: append_error_message_63547788 field: error.message value: '{{{_ingest.on_failure_message}}}' - fingerprint: + tag: fingerprint_02243b84 fields: - event.original target_field: _id ignore_missing: true - append: + tag: append_event_category_df23064f field: event.category value: email ignore_failure: true - append: + tag: append_event_type_61ba5509 field: event.type value: info ignore_failure: true - set: + tag: set_event_kind_de80643c field: event.kind value: event - convert: + tag: convert_json_senderIP_to_source_ip_0e7c10d3 field: json.senderIP target_field: source.ip type: ip ignore_missing: true on_failure: - remove: + tag: remove_json_senderIP_d3b08891 field: json.senderIP - append: + tag: append_error_message_18a6114e field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_related_ip_86c0561d field: related.ip value: '{{{source.ip}}}' if: ctx.source?.ip != null && ctx.source.ip != '' allow_duplicates: false ignore_failure: true - geoip: + tag: geoip_source_ip_to_source_geo_5d3ae3d6 field: source.ip target_field: source.geo ignore_missing: true if: ctx.source?.ip != null && ctx.source.ip != '' on_failure: - append: + tag: append_error_message_900765d9 field: error.message value: '{{{_ingest.on_failure_message}}}' - geoip: + tag: geoip_source_ip_to_source_as_82271077 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -93,17 +111,21 @@ processors: if: ctx.source?.ip != null && ctx.source.ip != '' on_failure: - append: + tag: append_error_message_e390fa84 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - foreach: + tag: foreach_json_messageParts_a01b3372 field: json.messageParts processor: append: @@ -114,6 +136,7 @@ processors: ignore_failure: true if: ctx.json?.messageParts instanceof List - foreach: + tag: foreach_json_messageParts_255f7bfb field: json.messageParts processor: append: @@ -124,55 +147,68 @@ processors: ignore_failure: true if: ctx.json?.messageParts instanceof List - rename: + tag: rename_json_ccAddresses_to_email_cc_address_11b02c18 field: json.ccAddresses target_field: email.cc.address ignore_missing: true - set: + tag: set_email_cc_address_76afae5b field: email.cc.address value: ['{{{email.cc.address}}}'] if: ctx.email?.cc?.address instanceof String - set: + tag: set_email_delivery_timestamp_8dc48ca9 field: email.delivery_timestamp copy_from: '@timestamp' ignore_failure: true - rename: + tag: rename_json_fromAddress_to_email_from_address_54a1d768 field: json.fromAddress target_field: email.from.address ignore_missing: true - set: + tag: set_email_from_address_fff7c0a5 field: email.from.address value: ['{{{email.from.address}}}'] if: ctx.email?.from?.address instanceof String - rename: + tag: rename_json_messageID_to_email_message_id_478a43db field: json.messageID target_field: email.message_id ignore_missing: true - gsub: + tag: gsub_email_message_id_9c7cf5ea field: email.message_id pattern: '<|>' replacement: '' ignore_missing: true - rename: + tag: rename_json_replyToAddress_to_email_reply_to_address_f55651c7 field: json.replyToAddress target_field: email.reply_to.address ignore_missing: true - rename: + tag: rename_json_sender_to_email_sender_address_90c69416 field: json.sender target_field: email.sender.address ignore_missing: true - rename: + tag: rename_json_subject_to_email_subject_dcbc667a field: json.subject target_field: email.subject ignore_missing: true - set: + tag: set_email_to_address_00a46272 field: email.to.address copy_from: json.toAddresses ignore_failure: true - set: + tag: set_email_to_address_52631338 field: email.to.address value: ['{{{email.to.address}}}'] if: ctx.email?.to?.address instanceof String - foreach: + tag: foreach_json_recipient_c1c02972 field: json.recipient processor: append: @@ -183,18 +219,22 @@ processors: ignore_failure: true if: ctx.json?.recipient instanceof List - rename: + tag: rename_json_xmailer_to_email_x_mailer_7644d631 field: json.xmailer target_field: email.x_mailer ignore_missing: true - rename: + tag: rename_json_id_to_event_id_6438e920 field: json.id target_field: event.id ignore_missing: true - set: + tag: set_email_attachments_75c4fd67 field: email.attachments copy_from: json.messageParts ignore_failure: true - foreach: + tag: foreach_email_attachments_f22ed6f5 field: email.attachments processor: remove: @@ -206,6 +246,7 @@ processors: ignore_failure: true if: ctx.email?.attachments instanceof List - foreach: + tag: foreach_email_attachments_5e0aa7a8 field: email.attachments processor: rename: @@ -215,6 +256,7 @@ processors: ignore_failure: true if: ctx.email?.attachments instanceof List - foreach: + tag: foreach_email_attachments_88d690e8 field: email.attachments processor: rename: @@ -224,6 +266,7 @@ processors: ignore_failure: true if: ctx.email?.attachments instanceof List - foreach: + tag: foreach_email_attachments_1ec76ffe field: email.attachments processor: rename: @@ -233,6 +276,7 @@ processors: ignore_failure: true if: ctx.email?.attachments instanceof List - foreach: + tag: foreach_email_attachments_7d6b12e8 field: email.attachments processor: rename: @@ -242,6 +286,7 @@ processors: ignore_failure: true if: ctx.email?.attachments instanceof List - script: + tag: script_458cf990 description: Adding hash in related.hash from artifact field. lang: painless ignore_failure: true @@ -265,67 +310,83 @@ processors: } } - rename: + tag: rename_json_toAddresses_to_proofpoint_tap_message_delivered_to_addresses_4f89a45f field: json.toAddresses target_field: proofpoint_tap.message_delivered.to_addresses ignore_missing: true - rename: + tag: rename_json_recipient_to_proofpoint_tap_message_delivered_recipient_15b8a94a field: json.recipient target_field: proofpoint_tap.message_delivered.recipient ignore_missing: true - rename: + tag: rename_json_cluster_to_proofpoint_tap_message_delivered_cluster_cd00a18e field: json.cluster target_field: proofpoint_tap.message_delivered.cluster ignore_missing: true - convert: + tag: convert_json_completelyRewritten_to_proofpoint_tap_message_delivered_completely_rewritten_97bb460c field: json.completelyRewritten target_field: proofpoint_tap.message_delivered.completely_rewritten type: string ignore_missing: true ignore_failure: true - rename: + tag: rename_json_GUID_to_proofpoint_tap_guid_ab57fad4 field: json.GUID target_field: proofpoint_tap.guid ignore_missing: true - rename: + tag: rename_json_headerFrom_to_proofpoint_tap_message_delivered_header_from_903299b8 field: json.headerFrom target_field: proofpoint_tap.message_delivered.header.from ignore_missing: true - gsub: + tag: gsub_proofpoint_tap_message_delivered_header_from_1e2441d6 field: proofpoint_tap.message_delivered.header.from pattern: '<|>' replacement: '' ignore_missing: true - rename: + tag: rename_json_headerReplyTo_to_proofpoint_tap_message_delivered_header_replyto_57f913f2 field: json.headerReplyTo target_field: proofpoint_tap.message_delivered.header.replyto ignore_missing: true - convert: + tag: convert_json_impostorScore_to_proofpoint_tap_message_delivered_impostor_score_c25af47d field: json.impostorScore target_field: proofpoint_tap.message_delivered.impostor_score type: double ignore_missing: true on_failure: - remove: + tag: remove_json_impostorScore_c936304e field: json.impostorScore - append: + tag: append_error_message_5c18a0a8 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_malwareScore_to_proofpoint_tap_message_delivered_malware_score_beb81838 field: json.malwareScore target_field: proofpoint_tap.message_delivered.malware_score type: long ignore_missing: true on_failure: - remove: + tag: remove_json_malwareScore_59616835 field: json.malwareScore - append: + tag: append_error_message_aa526cc3 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_messageParts_to_proofpoint_tap_message_delivered_message_parts_cddaa99b field: json.messageParts target_field: proofpoint_tap.message_delivered.message_parts ignore_missing: true - foreach: + tag: foreach_proofpoint_tap_message_delivered_message_parts_2649d356 field: proofpoint_tap.message_delivered.message_parts processor: remove: @@ -338,6 +399,7 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_delivered?.message_parts instanceof List - foreach: + tag: foreach_proofpoint_tap_message_delivered_message_parts_bd1ee55b field: proofpoint_tap.message_delivered.message_parts processor: rename: @@ -347,6 +409,7 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_delivered?.message_parts instanceof List - foreach: + tag: foreach_proofpoint_tap_message_delivered_message_parts_d79c3518 field: proofpoint_tap.message_delivered.message_parts processor: rename: @@ -356,63 +419,79 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_delivered?.message_parts instanceof List - convert: + tag: convert_json_messageSize_to_proofpoint_tap_message_delivered_message_size_576a5cdf field: json.messageSize target_field: proofpoint_tap.message_delivered.message_size type: long ignore_missing: true on_failure: - remove: + tag: remove_json_messageSize_074aa099 field: json.messageSize - append: + tag: append_error_message_f578818a field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_modulesRun_to_proofpoint_tap_message_delivered_modules_run_4761c49b field: json.modulesRun target_field: proofpoint_tap.message_delivered.modules_run ignore_missing: true - convert: + tag: convert_json_phishScore_to_proofpoint_tap_message_delivered_phish_score_51a7b42b field: json.phishScore target_field: proofpoint_tap.message_delivered.phish_score type: long ignore_missing: true on_failure: - remove: + tag: remove_json_phishScore_02488087 field: json.phishScore - append: + tag: append_error_message_c3e897f8 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_policyRoutes_to_proofpoint_tap_message_delivered_policy_routes_5020e58b field: json.policyRoutes target_field: proofpoint_tap.message_delivered.policy_routes ignore_missing: true - rename: + tag: rename_json_QID_to_proofpoint_tap_message_delivered_qid_2b35fec2 field: json.QID target_field: proofpoint_tap.message_delivered.qid ignore_missing: true - rename: + tag: rename_json_quarantineFolder_to_proofpoint_tap_message_delivered_quarantine_folder_3f86b282 field: json.quarantineFolder target_field: proofpoint_tap.message_delivered.quarantine.folder ignore_missing: true - rename: + tag: rename_json_quarantineRule_to_proofpoint_tap_message_delivered_quarantine_rule_77ab966a field: json.quarantineRule target_field: proofpoint_tap.message_delivered.quarantine.rule ignore_missing: true - convert: + tag: convert_json_spamScore_to_proofpoint_tap_message_delivered_spam_score_fc738360 field: json.spamScore target_field: proofpoint_tap.message_delivered.spam_score type: long ignore_missing: true on_failure: - remove: + tag: remove_json_spamScore_df8b477f field: json.spamScore - append: + tag: append_error_message_2ec6775d field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_threatsInfoMap_to_proofpoint_tap_message_delivered_threat_info_map_d3afb92f field: json.threatsInfoMap target_field: proofpoint_tap.message_delivered.threat_info_map ignore_missing: true - foreach: + tag: foreach_proofpoint_tap_message_delivered_threat_info_map_18ec0090 field: proofpoint_tap.message_delivered.threat_info_map processor: rename: @@ -422,6 +501,7 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_delivered?.threat_info_map instanceof List - foreach: + tag: foreach_proofpoint_tap_message_delivered_threat_info_map_2acfbdc7 field: proofpoint_tap.message_delivered.threat_info_map processor: rename: @@ -431,6 +511,7 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_delivered?.threat_info_map instanceof List - foreach: + tag: foreach_proofpoint_tap_message_delivered_threat_info_map_cd528d2d field: proofpoint_tap.message_delivered.threat_info_map processor: rename: @@ -440,6 +521,7 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_delivered?.threat_info_map instanceof List - foreach: + tag: foreach_proofpoint_tap_message_delivered_threat_info_map_4854e8a5 field: proofpoint_tap.message_delivered.threat_info_map processor: rename: @@ -449,6 +531,7 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_delivered?.threat_info_map instanceof List - foreach: + tag: foreach_proofpoint_tap_message_delivered_threat_info_map_92a641f9 field: proofpoint_tap.message_delivered.threat_info_map processor: date: @@ -460,6 +543,7 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_delivered?.threat_info_map instanceof List - foreach: + tag: foreach_proofpoint_tap_message_delivered_threat_info_map_d4f614a1 field: proofpoint_tap.message_delivered.threat_info_map processor: remove: @@ -468,6 +552,7 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_delivered?.threat_info_map instanceof List - foreach: + tag: foreach_proofpoint_tap_message_delivered_threat_info_map_0063be59 field: proofpoint_tap.message_delivered.threat_info_map processor: rename: @@ -477,6 +562,7 @@ processors: ignore_failure: true if: ctx.proofpoint_tap?.message_delivered?.threat_info_map instanceof List - foreach: + tag: foreach_proofpoint_tap_message_delivered_threat_info_map_52b25947 field: proofpoint_tap.message_delivered.threat_info_map processor: rename: @@ -499,10 +585,12 @@ processors: } } - remove: + tag: remove_e5a80934 field: - json ignore_missing: true - script: + tag: script_e2ccbdd7 description: Drops null/empty values recursively. lang: painless source: | diff --git a/packages/proofpoint_tap/manifest.yml b/packages/proofpoint_tap/manifest.yml index 3134226cdfb..bbf1891cf90 100644 --- a/packages/proofpoint_tap/manifest.yml +++ b/packages/proofpoint_tap/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.3.2" name: proofpoint_tap title: Proofpoint TAP -version: "1.31.2" +version: "1.31.3" description: Collect logs from Proofpoint TAP with Elastic Agent. type: integration categories: diff --git a/packages/qualys_vmdr/changelog.yml b/packages/qualys_vmdr/changelog.yml index 8944993b613..27154264c8a 100644 --- a/packages/qualys_vmdr/changelog.yml +++ b/packages/qualys_vmdr/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "6.19.5" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "6.19.4" changes: - description: Fix deduplication in the asset_host_detection data stream. diff --git a/packages/qualys_vmdr/data_stream/asset_host_detection/elasticsearch/ingest_pipeline/default.yml b/packages/qualys_vmdr/data_stream/asset_host_detection/elasticsearch/ingest_pipeline/default.yml index e26f8f59f06..6eeaffc54eb 100644 --- a/packages/qualys_vmdr/data_stream/asset_host_detection/elasticsearch/ingest_pipeline/default.yml +++ b/packages/qualys_vmdr/data_stream/asset_host_detection/elasticsearch/ingest_pipeline/default.yml @@ -19,22 +19,27 @@ processors: # Set up tag logic for choosing which cloud metadata to include. - append: + tag: append_tags_ae765f40 field: tags value: ["elastic_cloud_data", "provider_cloud_data"] if: ctx.tags?.contains('both') == true - script: + tag: script_023c6838 lang: painless if: ctx.tags?.contains('both') == true source: ctx.tags.remove(ctx.tags.indexOf('both')); - script: + tag: script_cfa1c445 lang: painless if: ctx.cloud == null && ctx.tags?.contains('elastic_cloud_data') == true source: ctx.tags.remove(ctx.tags.indexOf('elastic_cloud_data')); - set: + tag: set__conf_want_provider_cloud_4bad817b field: _conf.want_provider_cloud value: true if: ctx.tags?.contains('provider_cloud_data') == true - remove: + tag: remove_cloud_ddf0b5c6 description: Remove elastic agent-provided cloud metadata fields if not requested. field: cloud if: ctx.tags != null && !ctx.tags.contains('elastic_cloud_data') @@ -55,6 +60,7 @@ processors: if: ctx.interval_start != null && ctx.interval_start != '' on_failure: - append: + tag: append_error_message_fc9d4046 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - terminate: @@ -87,6 +93,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_5d1a8568 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -97,6 +104,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4287166e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -198,6 +206,7 @@ processors: if: ctx.json?.NETWORK_ID != '' on_failure: - append: + tag: append_error_message_3087b4ed field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -307,6 +316,7 @@ processors: if: ctx.json?.IP != '' on_failure: - append: + tag: append_error_message_efbd8fcb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -324,6 +334,7 @@ processors: if: ctx.json?.IPV6 != '' on_failure: - append: + tag: append_error_message_7f9683b7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -347,6 +358,7 @@ processors: if: ctx.json?.LAST_PC_SCANNED_DATE != null && ctx.json.LAST_PC_SCANNED_DATE != '' on_failure: - append: + tag: append_error_message_0b6700ef field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -358,6 +370,7 @@ processors: if: ctx.json?.LAST_SCAN_DATETIME != null && ctx.json.LAST_SCAN_DATETIME != '' on_failure: - append: + tag: append_error_message_4a31119b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -369,6 +382,7 @@ processors: if: ctx.json?.LAST_VM_AUTH_SCANNED_DATE != null && ctx.json.LAST_VM_AUTH_SCANNED_DATE != '' on_failure: - append: + tag: append_error_message_f3d866e6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -380,6 +394,7 @@ processors: if: ctx.json?.LAST_VM_SCANNED_DATE != null && ctx.json.LAST_VM_SCANNED_DATE != '' on_failure: - append: + tag: append_error_message_e17ffe1f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -391,6 +406,7 @@ processors: if: ctx.json?.LAST_VM_AUTH_SCANNED_DURATION != '' on_failure: - append: + tag: append_error_message_3b9265cd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -402,6 +418,7 @@ processors: if: ctx.json?.LAST_VM_SCANNED_DURATION != '' on_failure: - append: + tag: append_error_message_42cca5e7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -555,6 +572,7 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_bedb62c1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -578,6 +596,7 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_574e5e05 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -677,6 +696,7 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_ee6cda33 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -700,6 +720,7 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_cd632ee1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -799,6 +820,7 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_ecc930d7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -822,6 +844,7 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_6160e8e3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -921,6 +944,7 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_f9be3eff field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -944,6 +968,7 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_125373c5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1013,6 +1038,7 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_714475b9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -1168,6 +1194,7 @@ processors: } - set: + tag: set_cloud_instance_name_5969616d field: cloud.instance.name copy_from: qualys_vmdr.asset_host_detection.dns_data.hostname if: ctx._conf?.want_provider_cloud == true @@ -1347,6 +1374,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_326bd003 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1416,6 +1444,7 @@ processors: } on_failure: - append: + tag: append_error_message_80936042 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1464,6 +1493,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_bde8ad38 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1482,6 +1512,7 @@ processors: ignore_failure: true on_failure: - append: + tag: append_error_message_46c0fc2e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -1494,6 +1525,7 @@ processors: ignore_failure: true on_failure: - append: + tag: append_error_message_b2f9b220 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -1506,6 +1538,7 @@ processors: ignore_failure: true on_failure: - append: + tag: append_error_message_a2604ef8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -1517,6 +1550,7 @@ processors: ignore_failure: true on_failure: - append: + tag: append_error_message_302d664f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -1528,6 +1562,7 @@ processors: ignore_failure: true on_failure: - append: + tag: append_error_message_0ff3df5f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -1540,6 +1575,7 @@ processors: ignore_failure: true on_failure: - append: + tag: append_error_message_b941a75a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -1552,6 +1588,7 @@ processors: ignore_failure: true on_failure: - append: + tag: append_error_message_724755fc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -1564,6 +1601,7 @@ processors: ignore_failure: true on_failure: - append: + tag: append_error_message_60763118 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1575,6 +1613,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d5834489 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1586,6 +1625,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_98f742b7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1597,6 +1637,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_fb015da7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1608,6 +1649,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f6ae2a11 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1619,6 +1661,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2bd8c4f9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1630,6 +1673,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_def4adc1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1704,6 +1748,7 @@ processors: if: ctx.vulnerability?.score?.version == null on_failure: - append: + tag: append_error_message_e319b8f2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1714,8 +1759,10 @@ processors: if: ctx.vulnerability?.score?.base != '' on_failure: - remove: + tag: remove_vulnerability_score_base_a5b78b60 field: vulnerability.score.base - append: + tag: append_error_message_c4827698 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1750,15 +1797,18 @@ processors: } on_failure: - append: + tag: append_error_message_d717ec5f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_message_to_event_original_86cdcc50 field: message target_field: event.original if: ctx.event?.original == null ignore_missing: true ignore_failure: true - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: ctx.event?.original != null @@ -1773,6 +1823,7 @@ processors: - event.original target_field: _id - set: + tag: set__id_f1652dc2 description: > Prefix the _id with the scan date time because identifiers that generally increase over time based on sorting order generally result in better @@ -1841,6 +1892,7 @@ processors: drop(ctx); on_failure: - append: + tag: append_error_message_2dc525b4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: diff --git a/packages/qualys_vmdr/data_stream/asset_host_detection/elasticsearch/ingest_pipeline/pipeline_knowledge_base.yml b/packages/qualys_vmdr/data_stream/asset_host_detection/elasticsearch/ingest_pipeline/pipeline_knowledge_base.yml index d5ac88b4cca..d04ce7141f5 100644 --- a/packages/qualys_vmdr/data_stream/asset_host_detection/elasticsearch/ingest_pipeline/pipeline_knowledge_base.yml +++ b/packages/qualys_vmdr/data_stream/asset_host_detection/elasticsearch/ingest_pipeline/pipeline_knowledge_base.yml @@ -47,7 +47,7 @@ processors: tag: rename_COMPLIANCE_LIST_COMPLIANCE_DESCRIPTION target_field: qualys_vmdr.asset_host_detection.knowledge_base.compliance_list.description ignore_missing: true -# Handle case when COMPLIANCE_LIST.COMPLIANCE is an array + # Handle case when COMPLIANCE_LIST.COMPLIANCE is an array - foreach: field: qualys_vmdr.asset_host_detection.knowledge_base.COMPLIANCE_LIST.COMPLIANCE if: ctx.qualys_vmdr?.asset_host_detection?.knowledge_base?.COMPLIANCE_LIST?.COMPLIANCE instanceof List @@ -145,7 +145,7 @@ processors: tag: set_vulnerability_title copy_from: qualys_vmdr.asset_host_detection.knowledge_base.title ignore_empty_value: true -# Handle case when BUGTRAQ_LIST.BUGTRAQ is an object + # Handle case when BUGTRAQ_LIST.BUGTRAQ is an object - rename: field: qualys_vmdr.asset_host_detection.knowledge_base.BUGTRAQ_LIST.BUGTRAQ.ID tag: rename_BUGTRAQ_LIST_BUGTRAQ_ID @@ -156,7 +156,7 @@ processors: tag: rename_BUGTRAQ_LIST_BUGTRAQ_URL target_field: qualys_vmdr.asset_host_detection.knowledge_base.bugtraq_list.url ignore_missing: true -# Handle case when BUGTRAQ_LIST.BUGTRAQ is an array + # Handle case when BUGTRAQ_LIST.BUGTRAQ is an array - foreach: field: qualys_vmdr.asset_host_detection.knowledge_base.BUGTRAQ_LIST.BUGTRAQ if: ctx.qualys_vmdr?.asset_host_detection?.knowledge_base?.BUGTRAQ_LIST?.BUGTRAQ instanceof List @@ -201,7 +201,7 @@ processors: if: ctx.qualys_vmdr?.asset_host_detection?.knowledge_base?.CVSS?.BASE instanceof String - rename: field: qualys_vmdr.asset_host_detection.knowledge_base.CVSS.BASE - tag: rename_CVSS_BASE + tag: rename_qualys_vmdr_asset_host_detection_knowledge_base_CVSS_BASE_to_qualys_vmdr_asset_host_detection_knowledge_base_cvss_base_obj_b187039b target_field: qualys_vmdr.asset_host_detection.knowledge_base.cvss.base_obj ignore_missing: true if: ctx.qualys_vmdr?.asset_host_detection?.knowledge_base?.CVSS?.BASE instanceof Object @@ -372,6 +372,7 @@ processors: if: ctx.vulnerability?.reference != null && !(ctx.vulnerability.reference instanceof List) on_failure: - append: + tag: append_error_message_ccfa865f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' # Handle case when EXPLT_SRC and EXPLT_LIST.EXPLT is an object @@ -521,6 +522,7 @@ processors: if: ctx.qualys_vmdr?.asset_host_detection?.knowledge_base?.DISCOVERY?.REMOTE != '' on_failure: - append: + tag: append_error_message_b50b9071 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' # Handle case when SOFTWARE_LIST.SOFTWARE is an object @@ -608,6 +610,7 @@ processors: if: ctx.qualys_vmdr?.asset_host_detection?.knowledge_base?.LAST_SERVICE_MODIFICATION_DATETIME != null && ctx.qualys_vmdr.asset_host_detection.knowledge_base.LAST_SERVICE_MODIFICATION_DATETIME != '' on_failure: - append: + tag: append_error_message_c400757d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -619,6 +622,7 @@ processors: if: ctx.qualys_vmdr?.asset_host_detection?.knowledge_base?.LAST_CUSTOMIZATION?.DATETIME != null && ctx.qualys_vmdr.asset_host_detection.knowledge_base.LAST_CUSTOMIZATION.DATETIME != '' on_failure: - append: + tag: append_error_message_04594c67 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -635,6 +639,7 @@ processors: if: ctx.qualys_vmdr?.asset_host_detection?.knowledge_base?.PUBLISHED_DATETIME != null && ctx.qualys_vmdr.asset_host_detection.knowledge_base.PUBLISHED_DATETIME != '' on_failure: - append: + tag: append_error_message_a8d87209 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -646,6 +651,7 @@ processors: if: ctx.qualys_vmdr?.asset_host_detection?.knowledge_base?.PATCH_PUBLISHED_DATE != null && ctx.qualys_vmdr.asset_host_detection.knowledge_base.PATCH_PUBLISHED_DATE != '' on_failure: - append: + tag: append_error_message_a59d94f7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' # Handle case when CHANGE_LOG_LIST.CHANGE_LOG_INFO is an object @@ -668,6 +674,7 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_1a644f5a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' # Handle case when CHANGE_LOG_LIST.CHANGE_LOG_INFO is an array @@ -741,6 +748,7 @@ processors: if: ctx.qualys_vmdr?.asset_host_detection?.knowledge_base?.IS_DISABLED != '' on_failure: - append: + tag: append_error_message_eb552e7d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -762,6 +770,7 @@ processors: if: ctx.qualys_vmdr?.asset_host_detection?.knowledge_base?.PATCHABLE != '' on_failure: - append: + tag: append_error_message_293678bd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -783,6 +792,7 @@ processors: if: ctx.qualys_vmdr?.asset_host_detection?.knowledge_base?.PCI_FLAG != '' on_failure: - append: + tag: append_error_message_b0e6a94d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -816,6 +826,7 @@ processors: } on_failure: - append: + tag: append_error_message_7e49dfaa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -864,6 +875,7 @@ processors: drop(ctx); on_failure: - append: + tag: append_error_message_2dc525b4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: diff --git a/packages/qualys_vmdr/data_stream/knowledge_base/elasticsearch/ingest_pipeline/default.yml b/packages/qualys_vmdr/data_stream/knowledge_base/elasticsearch/ingest_pipeline/default.yml index fda5f4e8229..4a6e4608f55 100644 --- a/packages/qualys_vmdr/data_stream/knowledge_base/elasticsearch/ingest_pipeline/default.yml +++ b/packages/qualys_vmdr/data_stream/knowledge_base/elasticsearch/ingest_pipeline/default.yml @@ -39,9 +39,11 @@ processors: if: ctx.message != null on_failure: - append: + tag: append_error_message_ca675c68 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - fingerprint: + tag: fingerprint_aa6e5543 fields: - json.QID - json.LAST_SERVICE_MODIFICATION_DATETIME @@ -89,7 +91,7 @@ processors: tag: rename_COMPLIANCE_LIST_COMPLIANCE_DESCRIPTION target_field: qualys_vmdr.knowledge_base.compliance_list.description ignore_missing: true -# Handle case when COMPLIANCE_LIST.COMPLIANCE is an array + # Handle case when COMPLIANCE_LIST.COMPLIANCE is an array - foreach: field: json.COMPLIANCE_LIST.COMPLIANCE if: ctx.json?.COMPLIANCE_LIST?.COMPLIANCE instanceof List @@ -182,7 +184,7 @@ processors: tag: rename_TITLE target_field: qualys_vmdr.knowledge_base.title ignore_missing: true -# Handle case when BUGTRAQ_LIST.BUGTRAQ is an object + # Handle case when BUGTRAQ_LIST.BUGTRAQ is an object - rename: field: json.BUGTRAQ_LIST.BUGTRAQ.ID tag: rename_BUGTRAQ_LIST_BUGTRAQ_ID @@ -193,7 +195,7 @@ processors: tag: rename_BUGTRAQ_LIST_BUGTRAQ_URL target_field: qualys_vmdr.knowledge_base.bugtraq_list.url ignore_missing: true -# Handle case when BUGTRAQ_LIST.BUGTRAQ is an array + # Handle case when BUGTRAQ_LIST.BUGTRAQ is an array - foreach: field: json.BUGTRAQ_LIST.BUGTRAQ if: ctx.json?.BUGTRAQ_LIST?.BUGTRAQ instanceof List @@ -238,7 +240,7 @@ processors: if: ctx.json?.CVSS?.BASE instanceof String - rename: field: json.CVSS.BASE - tag: rename_CVSS_BASE + tag: rename_json_CVSS_BASE_to_qualys_vmdr_knowledge_base_cvss_base_obj_31654f88 target_field: qualys_vmdr.knowledge_base.cvss.base_obj ignore_missing: true if: ctx.json?.CVSS?.BASE instanceof Object @@ -388,6 +390,7 @@ processors: if: ctx.vulnerability?.reference != null on_failure: - append: + tag: append_error_message_8e74ba55 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -469,6 +472,7 @@ processors: if: ctx.json?.DISCOVERY?.REMOTE != '' on_failure: - append: + tag: append_error_message_1d343a40 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' # Handle case when SOFTWARE_LIST.SOFTWARE is an object @@ -556,6 +560,7 @@ processors: if: ctx.json?.LAST_SERVICE_MODIFICATION_DATETIME != null && ctx.json.LAST_SERVICE_MODIFICATION_DATETIME != '' on_failure: - append: + tag: append_error_message_689c6b4c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -566,6 +571,7 @@ processors: if: ctx.json?.LAST_SERVICE_MODIFICATION_DATETIME != null && ctx.json.LAST_SERVICE_MODIFICATION_DATETIME != '' on_failure: - append: + tag: append_error_message_791c8a56 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -577,6 +583,7 @@ processors: if: ctx.json?.LAST_CUSTOMIZATION?.DATETIME != null && ctx.json.LAST_CUSTOMIZATION.DATETIME != '' on_failure: - append: + tag: append_error_message_b1b2616a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -593,6 +600,7 @@ processors: if: ctx.json?.PUBLISHED_DATETIME != null && ctx.json.PUBLISHED_DATETIME != '' on_failure: - append: + tag: append_error_message_138ef15a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -604,6 +612,7 @@ processors: if: ctx.json?.PATCH_PUBLISHED_DATE != null && ctx.json.PATCH_PUBLISHED_DATE != '' on_failure: - append: + tag: append_error_message_fdb4d3a4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' # Handle case when CHANGE_LOG_LIST.CHANGE_LOG_INFO is an object @@ -626,6 +635,7 @@ processors: - ISO8601 on_failure: - append: + tag: append_error_message_61efb70a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' # Handle case when CHANGE_LOG_LIST.CHANGE_LOG_INFO is an array @@ -699,6 +709,7 @@ processors: if: ctx.json?.IS_DISABLED != '' on_failure: - append: + tag: append_error_message_8205c654 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -720,6 +731,7 @@ processors: if: ctx.json?.PATCHABLE != '' on_failure: - append: + tag: append_error_message_6c0c2c08 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -741,6 +753,7 @@ processors: if: ctx.json?.PCI_FLAG != '' on_failure: - append: + tag: append_error_message_d5e8837a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -784,6 +797,7 @@ processors: } on_failure: - append: + tag: append_error_message_0945afe3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -792,12 +806,14 @@ processors: copy_from: json.SEVERITY_LEVEL ignore_empty_value: true - rename: + tag: rename_message_to_event_original_86cdcc50 field: message target_field: event.original if: ctx.event?.original == null ignore_missing: true ignore_failure: true - remove: + tag: remove_message_2ee3a7d4 field: message ignore_missing: true if: ctx.event?.original != null @@ -842,6 +858,7 @@ processors: drop(ctx); on_failure: - append: + tag: append_error_message_2dc525b4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -872,6 +889,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/qualys_vmdr/data_stream/user_activity/elasticsearch/ingest_pipeline/default.yml b/packages/qualys_vmdr/data_stream/user_activity/elasticsearch/ingest_pipeline/default.yml index dd9a354fa9e..157cbae16b5 100644 --- a/packages/qualys_vmdr/data_stream/user_activity/elasticsearch/ingest_pipeline/default.yml +++ b/packages/qualys_vmdr/data_stream/user_activity/elasticsearch/ingest_pipeline/default.yml @@ -2,6 +2,7 @@ description: Pipeline for processing Qualys User Activity Log data. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: 8.11.0 - terminate: @@ -20,15 +21,18 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - remove: + tag: remove_message_691cf1d4 if: ctx.event?.original != null field: message ignore_missing: true - json: + tag: json_event_original_to_qualys_vmdr_user_activity_7d6ad8d0 field: event.original target_field: qualys_vmdr.user_activity - script: @@ -47,50 +51,62 @@ processors: } ctx.qualys_vmdr.user_activity = out; - fingerprint: + tag: fingerprint_eaed5dd9 fields: - event.original target_field: _id - date: + tag: date_qualys_vmdr_user_activity_Date_ee34d4e3 field: qualys_vmdr.user_activity.Date formats: - ISO8601 - set: + tag: set_event_action_71826743 field: event.action copy_from: qualys_vmdr.user_activity.Action ignore_empty_value: true - set: + tag: set_event_provider_dfe42be4 field: event.provider copy_from: qualys_vmdr.user_activity.Module ignore_empty_value: true - set: + tag: set_message_f0ade48a field: message copy_from: qualys_vmdr.user_activity.Details ignore_empty_value: true - set: + tag: set_user_name_4ff0b4e0 field: user.name copy_from: qualys_vmdr.user_activity.User_Name ignore_empty_value: true - append: + tag: append_related_user_c036eec2 if: ctx.user?.name != null field: related.user value: '{{{user.name}}}' - append: + tag: append_user_roles_fbe29edc field: user.roles value: '{{{qualys_vmdr.user_activity.User_Role}}}' - convert: + tag: convert_qualys_vmdr_user_activity_User_IP_to_source_ip_a1df5e65 field: qualys_vmdr.user_activity.User_IP target_field: source.ip type: ip ignore_failure: true - append: + tag: append_related_ip_86933b04 if: ctx.source?.ip != null field: related.ip value: '{{{source.ip}}}' - geoip: + tag: geoip_source_ip_to_source_geo_da2e41b2 field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -99,10 +115,12 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true @@ -119,41 +137,50 @@ processors: if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) ignore_missing: true - remove: + tag: remove_qualys_vmdr_user_activity_e2016f02 if: ctx?.qualys_vmdr?.user_activity instanceof Map && ctx.qualys_vmdr.user_activity.isEmpty() field: qualys_vmdr.user_activity - remove: + tag: remove_qualys_vmdr_e6196be2 if: ctx?.qualys_vmdr instanceof Map && ctx.qualys_vmdr.isEmpty() field: qualys_vmdr - remove: + tag: remove_a7d4f1b2 description: Remove metadata unrelated to the user activity log. field: - cloud - host ignore_missing: true - set: + tag: set_event_kind_de80643c field: event.kind value: event - append: + tag: append_event_category_baca8d41 if: ctx.event?.action == "login" field: event.category value: authentication allow_duplicates: false - append: + tag: append_event_type_23a63b39 if: ctx.event?.action == "login" field: event.type value: info allow_duplicates: false - append: + tag: append_event_category_0de26197 if: ctx.event?.action == "request" field: event.category value: api allow_duplicates: false - append: + tag: append_event_type_945ac10d if: ctx.event?.action == "request" field: event.type value: info allow_duplicates: false - append: + tag: append_event_category_c7e1f7cd if: > ctx.event?.action == 'add' || ctx.event?.action == 'create' @@ -162,6 +189,7 @@ processors: value: configuration allow_duplicates: false - append: + tag: append_event_type_09e4c29d if: > ctx.event?.action == 'add' || ctx.event?.action == 'set' @@ -169,6 +197,7 @@ processors: value: change allow_duplicates: false - append: + tag: append_event_type_82284c0b if: ctx.event?.action == 'create' field: event.type value: creation diff --git a/packages/qualys_vmdr/manifest.yml b/packages/qualys_vmdr/manifest.yml index 22880d5a64f..58a122ddb3b 100644 --- a/packages/qualys_vmdr/manifest.yml +++ b/packages/qualys_vmdr/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.4.0" name: qualys_vmdr title: Qualys VMDR -version: "6.19.4" +version: "6.19.5" description: Collect data from Qualys VMDR platform with Elastic Agent. type: integration categories: diff --git a/packages/rapid7_insightvm/changelog.yml b/packages/rapid7_insightvm/changelog.yml index 0f9b4fe9abc..d6683884d6b 100644 --- a/packages/rapid7_insightvm/changelog.yml +++ b/packages/rapid7_insightvm/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.9.3" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "2.9.2" changes: - description: Add missing field definition for rapid7.insightvm.asset.same.reintroduced. diff --git a/packages/rapid7_insightvm/data_stream/asset/elasticsearch/ingest_pipeline/default.yml b/packages/rapid7_insightvm/data_stream/asset/elasticsearch/ingest_pipeline/default.yml index 766abd731ed..fa2750998c3 100644 --- a/packages/rapid7_insightvm/data_stream/asset/elasticsearch/ingest_pipeline/default.yml +++ b/packages/rapid7_insightvm/data_stream/asset/elasticsearch/ingest_pipeline/default.yml @@ -2,15 +2,19 @@ description: Pipeline for processing Rapid7 InsightVM Asset logs. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - set: + tag: set_event_kind_871e09cf field: event.kind value: state - set: + tag: set_event_category_ba9b1d84 field: event.category value: [host] - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] - remove: @@ -43,9 +47,11 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_8075efaa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - drop: + tag: drop_eb865603 if: ctx.json?.data != null && ctx.json.data.isEmpty() - convert: field: json.assessed_for_policies @@ -56,6 +62,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f701e2fd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -67,9 +74,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_76f140e9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_credential_assessments_cd0df090 field: json.credential_assessments if: ctx.json?.credential_assessments instanceof List processor: @@ -86,6 +95,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_credential_assessments_to_rapid7_insightvm_asset_credential_assessments_e258c4c4 field: json.credential_assessments target_field: rapid7.insightvm.asset.credential_assessments ignore_missing: true @@ -95,6 +105,7 @@ processors: copy_from: rapid7.insightvm.asset.credential_assessments ignore_empty_value: true - foreach: + tag: foreach_rapid7_insightvm_asset_credential_assessments_ea15f572 field: rapid7.insightvm.asset.credential_assessments if: ctx.rapid7?.insightvm?.asset?.credential_assessments instanceof List processor: @@ -103,6 +114,7 @@ processors: value: '{{{_ingest._value.protocol}}}' allow_duplicates: false - foreach: + tag: foreach_network_transport_3bbf5841 field: network.transport if: ctx.network?.transport instanceof List processor: @@ -118,6 +130,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_80c9d3ad field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -129,25 +142,31 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_1cc4d685 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_host_name_to_rapid7_insightvm_asset_host_name_db97d98c field: json.host_name target_field: rapid7.insightvm.asset.host_name ignore_missing: true - set: + tag: set_host_name_725c4d2c field: host.name copy_from: rapid7.insightvm.asset.host_name ignore_empty_value: true - lowercase: + tag: lowercase_host_name_91ef3e1c field: host.name ignore_missing: true - append: + tag: append_related_hosts_452ef445 field: related.hosts value: '{{{host.name}}}' if: ctx.host?.name != null allow_duplicates: false - script: + tag: script_d8314f3a lang: painless if: ctx.host?.name != null source: | @@ -158,15 +177,18 @@ processors: ctx.host.hostname = ctx.host.name.substring(0, idx); } - append: + tag: append_related_hosts_ca923905 field: related.hosts value: '{{{host.hostname}}}' if: ctx.host?.hostname != null allow_duplicates: false - rename: + tag: rename_json_id_to_rapid7_insightvm_asset_id_e3c23c24 field: json.id target_field: rapid7.insightvm.asset.id ignore_missing: true - set: + tag: set_host_id_f2edcb27 field: host.id copy_from: rapid7.insightvm.asset.id ignore_empty_value: true @@ -179,14 +201,17 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_420c2701 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: + tag: append_host_ip_9f5f74e8 field: host.ip value: '{{{rapid7.insightvm.asset.ip}}}' if: ctx.rapid7?.insightvm?.asset?.ip != null allow_duplicates: false - append: + tag: append_related_ip_8d1b03a9 field: related.ip value: '{{{rapid7.insightvm.asset.ip}}}' if: ctx.rapid7?.insightvm?.asset?.ip != null @@ -200,6 +225,7 @@ processors: if: ctx.json?.last_assessed_for_vulnerabilities != null && ctx.json?.last_assessed_for_vulnerabilities != '' on_failure: - append: + tag: append_error_message_4e35aa73 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -211,6 +237,7 @@ processors: if: ctx.json?.last_scan_end != null && ctx.json?.last_scan_end != '' on_failure: - append: + tag: append_error_message_4cdbb842 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -222,14 +249,17 @@ processors: if: ctx.json?.last_scan_start != null && ctx.json?.last_scan_start != '' on_failure: - append: + tag: append_error_message_afcf7087 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - gsub: + tag: gsub_json_mac_cff51717 field: json.mac pattern: '[:.]' replacement: '-' ignore_missing: true - uppercase: + tag: uppercase_json_mac_to_rapid7_insightvm_asset_mac_3442e1e2 field: json.mac target_field: rapid7.insightvm.asset.mac ignore_missing: true @@ -248,6 +278,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f2ccd42f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -259,9 +290,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2dad55d5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_new_3123a6a1 field: json.new if: ctx.json?.new instanceof List processor: @@ -275,6 +308,7 @@ processors: field: _ingest._value.first_found ignore_missing: true - foreach: + tag: foreach_json_new_b0879ee1 field: json.new if: ctx.json?.new instanceof List processor: @@ -288,6 +322,7 @@ processors: field: _ingest._value.last_found ignore_missing: true - foreach: + tag: foreach_json_new_06afb2e5 field: json.new if: ctx.json?.new instanceof List processor: @@ -304,6 +339,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_new_d2753b74 field: json.new if: ctx.json?.new instanceof List processor: @@ -312,6 +348,7 @@ processors: value: '{{{_ingest._value.vulnerability_id}}}' allow_duplicates: false - foreach: + tag: foreach_json_new_edf3ec50 field: json.new if: ctx.json?.new instanceof List processor: @@ -325,6 +362,7 @@ processors: target_field: _ingest._value.solution.fix ignore_missing: true - foreach: + tag: foreach_json_new_3a138bb1 field: json.new if: ctx.json?.new instanceof List processor: @@ -333,6 +371,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_new_357be2fa field: json.new if: ctx.json?.new instanceof List processor: @@ -341,6 +380,7 @@ processors: target_field: _ingest._value.solution.id ignore_missing: true - foreach: + tag: foreach_json_new_729b7cf0 field: json.new if: ctx.json?.new instanceof List processor: @@ -349,6 +389,7 @@ processors: target_field: _ingest._value.solution.summary ignore_missing: true - foreach: + tag: foreach_json_new_a690db06 field: json.new if: ctx.json?.new instanceof List processor: @@ -357,6 +398,7 @@ processors: target_field: _ingest._value.solution.type ignore_missing: true - foreach: + tag: foreach_json_new_6bd0a671 field: json.new if: ctx.json?.new instanceof List processor: @@ -364,62 +406,77 @@ processors: field: _ingest._value.solution_fix ignore_missing: true - rename: + tag: rename_json_new_to_rapid7_insightvm_asset_new_e61676f8 field: json.new target_field: rapid7.insightvm.asset.new ignore_missing: true - rename: + tag: rename_json_os_architecture_to_rapid7_insightvm_asset_os_architecture_965eedf9 field: json.os_architecture target_field: rapid7.insightvm.asset.os.architecture ignore_missing: true - set: + tag: set_host_architecture_87156b11 field: host.architecture copy_from: rapid7.insightvm.asset.os.architecture ignore_empty_value: true - rename: + tag: rename_json_os_description_to_rapid7_insightvm_asset_os_description_fdbdb05b field: json.os_description target_field: rapid7.insightvm.asset.os.description ignore_missing: true - set: + tag: set_host_os_full_6927a11c field: host.os.full copy_from: rapid7.insightvm.asset.os.description ignore_empty_value: true - rename: + tag: rename_json_os_family_to_rapid7_insightvm_asset_os_family_69d872a9 field: json.os_family target_field: rapid7.insightvm.asset.os.family ignore_missing: true - set: + tag: set_host_os_family_485bd887 field: host.os.family copy_from: rapid7.insightvm.asset.os.family ignore_empty_value: true - rename: + tag: rename_json_os_name_to_rapid7_insightvm_asset_os_name_bfbc674d field: json.os_name target_field: rapid7.insightvm.asset.os.name ignore_missing: true - set: + tag: set_host_os_name_be99a7c9 field: host.os.name copy_from: rapid7.insightvm.asset.os.name ignore_empty_value: true - rename: + tag: rename_json_os_system_name_to_rapid7_insightvm_asset_os_system_name_e7d4cefd field: json.os_system_name target_field: rapid7.insightvm.asset.os.system_name ignore_missing: true - rename: + tag: rename_json_os_type_to_rapid7_insightvm_asset_os_type_0c866a95 field: json.os_type target_field: rapid7.insightvm.asset.os.type ignore_missing: true - rename: + tag: rename_json_os_vendor_to_rapid7_insightvm_asset_os_vendor_f7f6e261 field: json.os_vendor target_field: rapid7.insightvm.asset.os.vendor ignore_missing: true - rename: + tag: rename_json_os_version_to_rapid7_insightvm_asset_os_version_9f28d223 field: json.os_version target_field: rapid7.insightvm.asset.os.version ignore_missing: true - set: + tag: set_host_os_version_05e82ae7 field: host.os.version ignore_empty_value: true copy_from: rapid7.insightvm.asset.os.version - foreach: + tag: foreach_json_remediated_6d6318fb field: json.remediated if: ctx.json?.remediated instanceof List processor: @@ -433,6 +490,7 @@ processors: field: _ingest._value.first_found ignore_missing: true - foreach: + tag: foreach_json_remediated_7dff561f field: json.remediated if: ctx.json?.remediated instanceof List processor: @@ -446,6 +504,7 @@ processors: field: _ingest._value.last_found ignore_missing: true - foreach: + tag: foreach_json_remediated_1793a251 field: json.remediated if: ctx.json?.remediated instanceof List processor: @@ -462,6 +521,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_remediated_c9d3e892 field: json.remediated if: ctx.json?.remediated instanceof List processor: @@ -470,6 +530,7 @@ processors: value: '{{{_ingest._value.vulnerability_id}}}' allow_duplicates: false - foreach: + tag: foreach_json_remediated_5317e16a field: json.remediated if: ctx.json?.remediated instanceof List processor: @@ -483,6 +544,7 @@ processors: target_field: _ingest._value.solution.fix ignore_missing: true - foreach: + tag: foreach_json_remediated_a53a5ba7 field: json.remediated if: ctx.json?.remediated instanceof List processor: @@ -491,6 +553,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_remediated_d271cfe4 field: json.remediated if: ctx.json?.remediated instanceof List processor: @@ -499,6 +562,7 @@ processors: target_field: _ingest._value.solution.id ignore_missing: true - foreach: + tag: foreach_json_remediated_d9199ae2 field: json.remediated if: ctx.json?.remediated instanceof List processor: @@ -507,6 +571,7 @@ processors: target_field: _ingest._value.solution.summary ignore_missing: true - foreach: + tag: foreach_json_remediated_d65b26d0 field: json.remediated if: ctx.json?.remediated instanceof List processor: @@ -515,6 +580,7 @@ processors: target_field: _ingest._value.solution.type ignore_missing: true - foreach: + tag: foreach_json_remediated_57359827 field: json.remediated if: ctx.json?.remediated instanceof List processor: @@ -522,6 +588,7 @@ processors: field: _ingest._value.solution_fix ignore_missing: true - rename: + tag: rename_json_remediated_to_rapid7_insightvm_asset_remediated_30174952 field: json.remediated target_field: rapid7.insightvm.asset.remediated ignore_missing: true @@ -534,13 +601,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_1d267327 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_host_risk_static_score_48eaad0f field: host.risk.static_score copy_from: rapid7.insightvm.asset.risk_score ignore_empty_value: true - foreach: + tag: foreach_json_same_e1b27f17 field: json.same if: ctx.json?.same instanceof List processor: @@ -554,6 +624,7 @@ processors: field: _ingest._value.first_found ignore_missing: true - foreach: + tag: foreach_json_same_193b26f3 field: json.same if: ctx.json?.same instanceof List processor: @@ -567,6 +638,7 @@ processors: field: _ingest._value.last_found ignore_missing: true - foreach: + tag: foreach_json_same_1549ab1e field: json.same if: ctx.json?.same instanceof List processor: @@ -580,6 +652,7 @@ processors: field: _ingest._value.reintroduced ignore_missing: true - foreach: + tag: foreach_json_same_b3e4f21b field: json.same if: ctx.json?.same instanceof List processor: @@ -596,6 +669,7 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_same_8686806e field: json.same if: ctx.json?.same instanceof List processor: @@ -604,6 +678,7 @@ processors: value: '{{{_ingest._value.vulnerability_id}}}' allow_duplicates: false - foreach: + tag: foreach_json_same_21dfd2ae field: json.same if: ctx.json?.same instanceof List processor: @@ -617,6 +692,7 @@ processors: target_field: _ingest._value.solution.fix ignore_missing: true - foreach: + tag: foreach_json_same_a163360b field: json.same if: ctx.json?.same instanceof List processor: @@ -625,6 +701,7 @@ processors: ignore_missing: true ignore_failure: true - foreach: + tag: foreach_json_same_8876f220 field: json.same if: ctx.json?.same instanceof List processor: @@ -633,6 +710,7 @@ processors: target_field: _ingest._value.solution.id ignore_missing: true - foreach: + tag: foreach_json_same_c8dfd11e field: json.same if: ctx.json?.same instanceof List processor: @@ -641,6 +719,7 @@ processors: target_field: _ingest._value.solution.summary ignore_missing: true - foreach: + tag: foreach_json_same_eaf89e7c field: json.same if: ctx.json?.same instanceof List processor: @@ -649,6 +728,7 @@ processors: target_field: _ingest._value.solution.type ignore_missing: true - foreach: + tag: foreach_json_same_e619aa73 field: json.same if: ctx.json?.same instanceof List processor: @@ -656,6 +736,7 @@ processors: field: _ingest._value.solution_fix ignore_missing: true - rename: + tag: rename_json_same_to_rapid7_insightvm_asset_same_327679ce field: json.same target_field: rapid7.insightvm.asset.same ignore_missing: true @@ -668,9 +749,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2875b6e7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tags_to_rapid7_insightvm_asset_tags_6c36824c field: json.tags target_field: rapid7.insightvm.asset.tags ignore_missing: true @@ -688,13 +771,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e319e88b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_type_to_rapid7_insightvm_asset_type_874c7736 field: json.type target_field: rapid7.insightvm.asset.type ignore_missing: true - rename: + tag: rename_json_unique_identifiers_to_rapid7_insightvm_asset_unique_identifiers_5c5e48b2 field: json.unique_identifiers target_field: rapid7.insightvm.asset.unique_identifiers ignore_missing: true @@ -704,9 +790,11 @@ processors: copy_from: rapid7.insightvm.asset.unique_identifiers ignore_empty_value: true - remove: + tag: remove_json_29cdffdc field: json ignore_missing: true - remove: + tag: remove_de46ffcb if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - rapid7.insightvm.asset.risk_score @@ -721,6 +809,7 @@ processors: - rapid7.insightvm.asset.os.version ignore_missing: true - foreach: + tag: foreach_rapid7_insightvm_asset_new_fec3635f field: rapid7.insightvm.asset.new if: ctx.rapid7?.insightvm?.asset?.new instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) ignore_failure: true @@ -730,6 +819,7 @@ processors: - _ingest._value.vulnerability_id ignore_missing: true - foreach: + tag: foreach_rapid7_insightvm_asset_remediated_15c97c49 field: rapid7.insightvm.asset.remediated if: ctx.rapid7?.insightvm?.asset?.remediated instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) ignore_failure: true @@ -739,6 +829,7 @@ processors: - _ingest._value.vulnerability_id ignore_missing: true - foreach: + tag: foreach_rapid7_insightvm_asset_same_e54396a5 field: rapid7.insightvm.asset.same if: ctx.rapid7?.insightvm?.asset?.same instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) ignore_failure: true @@ -748,6 +839,7 @@ processors: - _ingest._value.vulnerability_id ignore_missing: true - script: + tag: script_06971239 lang: painless description: Drops null/empty values recursively. source: |- @@ -765,10 +857,12 @@ processors: } drop(ctx); - set: + tag: set_event_kind_92954dfa field: event.kind value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/rapid7_insightvm/data_stream/asset_vulnerability/elasticsearch/ingest_pipeline/default.yml b/packages/rapid7_insightvm/data_stream/asset_vulnerability/elasticsearch/ingest_pipeline/default.yml index 606ba9f58aa..2db661bee9f 100644 --- a/packages/rapid7_insightvm/data_stream/asset_vulnerability/elasticsearch/ingest_pipeline/default.yml +++ b/packages/rapid7_insightvm/data_stream/asset_vulnerability/elasticsearch/ingest_pipeline/default.yml @@ -39,6 +39,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_cea45f41 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -76,6 +77,7 @@ processors: # Remove cloud.* fields populated by beat. # These fields correspond to EA rather than Rapid7 hosts and could be misleading. - remove: + tag: remove_cloud_2d3e08b6 field: cloud ignore_missing: true description: Remove ECS cloud fields that are populated from EA metadata. @@ -87,6 +89,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_294eb4a8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -97,6 +100,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_1e53f7ee field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -107,6 +111,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_657e0fad field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -132,6 +137,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_57f217b8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -142,6 +148,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_349350d0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -200,6 +207,7 @@ processors: if: ctx.json?.ip != '' on_failure: - append: + tag: append_error_message_42d0ee63 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: @@ -223,6 +231,7 @@ processors: if: ctx.json?.last_assessed_for_vulnerabilities != null && ctx.json.last_assessed_for_vulnerabilities != '' on_failure: - append: + tag: append_error_message_3e3d033c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -234,6 +243,7 @@ processors: if: ctx.json?.last_scan_end != null && ctx.json.last_scan_end != '' on_failure: - append: + tag: append_error_message_9bbf7b73 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -245,6 +255,7 @@ processors: if: ctx.json?.last_scan_start != null && ctx.json.last_scan_start != '' on_failure: - append: + tag: append_error_message_a52e8ae8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - gsub: @@ -272,6 +283,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_3913bc69 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -282,6 +294,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_35a681d0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -356,6 +369,7 @@ processors: } on_failure: - append: + tag: append_error_message_5540cddb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -401,6 +415,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e61a4080 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -416,6 +431,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_cd00da55 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -441,6 +457,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_92402fd3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -479,6 +496,7 @@ processors: } on_failure: - append: + tag: append_error_message_c3c80162 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -486,10 +504,11 @@ processors: tag: date_vulnerability_added target_field: rapid7_insightvm.asset_vulnerability.vulnerability.added formats: - - ISO8601 + - ISO8601 if: ctx.json?.vulnerability?.added != null && ctx.json.vulnerability.added != '' on_failure: - append: + tag: append_error_message_34aa1871 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - split: @@ -501,6 +520,7 @@ processors: if: ctx.json?.vulnerability?.categories instanceof String on_failure: - append: + tag: append_error_message_d48c3de2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -522,6 +542,7 @@ processors: if: ctx.json?.vulnerability?.cves instanceof String && ctx.json.vulnerability.cves != '' on_failure: - append: + tag: append_error_message_01bdab10 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -562,6 +583,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f54b9911 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -572,6 +594,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_cbbd6552 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -587,6 +610,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4fa9fbb5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -622,6 +646,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_acb75570 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -632,6 +657,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8aabcb83 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -657,6 +683,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f838123c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -686,6 +713,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_1bcbd934 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -712,6 +740,7 @@ processors: if: ctx.json?.vulnerability?.first_found != null && ctx.json.vulnerability.first_found != '' on_failure: - append: + tag: append_error_message_0cc21c46 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -743,6 +772,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_afcbecd4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -753,6 +783,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_145c1c5c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -764,6 +795,7 @@ processors: if: ctx.json?.vulnerability?.last_found != null && ctx.json.vulnerability.last_found != '' on_failure: - append: + tag: append_error_message_c38acf56 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -781,6 +813,7 @@ processors: target_field: rapid7_insightvm.asset_vulnerability.vulnerability.links ignore_missing: true - foreach: + tag: foreach_rapid7_insightvm_asset_vulnerability_vulnerability_links_6df19e68 field: rapid7_insightvm.asset_vulnerability.vulnerability.links if: ctx.rapid7_insightvm?.asset_vulnerability?.vulnerability?.links instanceof List processor: @@ -803,6 +836,7 @@ processors: if: ctx.json?.vulnerability?.modified != null && ctx.json.vulnerability.modified != '' on_failure: - append: + tag: append_error_message_e5c3dc42 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -813,6 +847,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2e45a83b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -823,6 +858,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_52ec88d3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -833,6 +869,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_011cb485 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -853,6 +890,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_6bc6aba0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - html_strip: @@ -862,6 +900,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_0beb89a2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - trim: @@ -870,6 +909,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f5562528 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -886,6 +926,7 @@ processors: if: ctx.json?.vulnerability?.published != null && ctx.json.vulnerability.published != '' on_failure: - append: + tag: append_error_message_d4237c1d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -906,6 +947,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5d90e99b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -935,6 +977,7 @@ processors: } on_failure: - append: + tag: append_error_message_9e2e8224 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -945,6 +988,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4c25eaa5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -959,6 +1003,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d0fa33a4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - trim: @@ -967,6 +1012,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_36cf1889 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1012,6 +1058,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5775cc5b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1030,6 +1077,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_49066c45 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -1064,6 +1112,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5b85f525 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: @@ -1126,6 +1175,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/rapid7_insightvm/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml b/packages/rapid7_insightvm/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml index 141cc22ac46..555e16a1b0f 100644 --- a/packages/rapid7_insightvm/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml +++ b/packages/rapid7_insightvm/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml @@ -2,15 +2,19 @@ description: Pipeline for processing Rapid7 InsightVM Vulnerability logs. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_category_13b30ef2 field: event.category value: [vulnerability] - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] - remove: @@ -43,11 +47,14 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_8075efaa field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - drop: + tag: drop_eb865603 if: ctx.json?.data != null && ctx.json.data.isEmpty() - fingerprint: + tag: fingerprint_3393286e fields: - json.id - json.modified @@ -66,17 +73,21 @@ processors: if: ctx.json?.added != null && ctx.json?.added != '' on_failure: - append: + tag: append_error_message_d2147744 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - - split: + - split: + tag: split_json_categories_3460790a field: json.categories separator: ',' if: ctx.json?.categories != null - rename: + tag: rename_json_categories_to_rapid7_insightvm_vulnerability_categories_88834680 field: json.categories target_field: rapid7.insightvm.vulnerability.categories ignore_missing: true - foreach: + tag: foreach_rapid7_insightvm_vulnerability_categories_f7adfa16 field: rapid7.insightvm.vulnerability.categories if: ctx.rapid7?.insightvm?.vulnerability?.categories instanceof List processor: @@ -90,42 +101,52 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_vulnerability_classification_8b34b171 field: vulnerability.classification value: 'CVSS' - set: + tag: set_vulnerability_scanner_vendor_1f10f1db field: vulnerability.scanner.vendor value: 'Rapid7' - set: + tag: set_vulnerability_enumeration_77e6939a field: vulnerability.enumeration value: 'CVE' if: ctx.json?.cves != null && ctx.json?.cves?.startsWith('CVE') - split: + tag: split_json_cves_to_rapid7_insightvm_vulnerability_cves_ff55c59a field: json.cves target_field: rapid7.insightvm.vulnerability.cves separator: ',' ignore_missing: true if: ctx.json?.cves != null && ctx.json.cves != '' - set: + tag: set_vulnerability_id_5651fba9 field: vulnerability.id copy_from: rapid7.insightvm.vulnerability.cves ignore_empty_value: true - rename: + tag: rename_json_cvss_v2_access_complexity_to_rapid7_insightvm_vulnerability_cvss_v2_access_complexity_9e8aa110 field: json.cvss_v2_access_complexity target_field: rapid7.insightvm.vulnerability.cvss.v2.access_complexity ignore_missing: true - rename: + tag: rename_json_cvss_v2_access_vector_to_rapid7_insightvm_vulnerability_cvss_v2_access_vector_89c6ef74 field: json.cvss_v2_access_vector target_field: rapid7.insightvm.vulnerability.cvss.v2.access_vector ignore_missing: true - rename: + tag: rename_json_cvss_v2_authentication_to_rapid7_insightvm_vulnerability_cvss_v2_authentication_c3064b74 field: json.cvss_v2_authentication target_field: rapid7.insightvm.vulnerability.cvss.v2.authentication ignore_missing: true - rename: + tag: rename_json_cvss_v2_availability_impact_to_rapid7_insightvm_vulnerability_cvss_v2_availability_impact_a7c9a818 field: json.cvss_v2_availability_impact target_field: rapid7.insightvm.vulnerability.cvss.v2.availability_impact ignore_missing: true - rename: + tag: rename_json_cvss_v2_confidentiality_impact_to_rapid7_insightvm_vulnerability_cvss_v2_confidentiality_impact_9bd3384a field: json.cvss_v2_confidentiality_impact target_field: rapid7.insightvm.vulnerability.cvss.v2.confidentiality_impact ignore_missing: true @@ -138,6 +159,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_3d0e1f43 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -149,9 +171,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_bda3959b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_cvss_v2_integrity_impact_to_rapid7_insightvm_vulnerability_cvss_v2_integrity_impact_2d09c030 field: json.cvss_v2_integrity_impact target_field: rapid7.insightvm.vulnerability.cvss.v2.integrity_impact ignore_missing: true @@ -164,30 +188,37 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d508f347 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: + tag: append_vulnerability_score_base_18c89660 field: vulnerability.score.base value: '{{{rapid7.insightvm.vulnerability.cvss.v2.score}}}' if: ctx.rapid7?.insightvm?.vulnerability?.cvss.v2.score != null allow_duplicates: false - rename: + tag: rename_json_cvss_v2_vector_to_rapid7_insightvm_vulnerability_cvss_v2_vector_77e3369a field: json.cvss_v2_vector target_field: rapid7.insightvm.vulnerability.cvss.v2.vector ignore_missing: true - rename: + tag: rename_json_cvss_v3_attack_complexity_to_rapid7_insightvm_vulnerability_cvss_v3_attack_complexity_750f911c field: json.cvss_v3_attack_complexity target_field: rapid7.insightvm.vulnerability.cvss.v3.attack_complexity ignore_missing: true - rename: + tag: rename_json_cvss_v3_attack_vector_to_rapid7_insightvm_vulnerability_cvss_v3_attack_vector_b66b27c8 field: json.cvss_v3_attack_vector target_field: rapid7.insightvm.vulnerability.cvss.v3.attack_vector ignore_missing: true - rename: + tag: rename_json_cvss_v3_availability_impact_to_rapid7_insightvm_vulnerability_cvss_v3_availability_impact_c7ddc32c field: json.cvss_v3_availability_impact target_field: rapid7.insightvm.vulnerability.cvss.v3.availability_impact ignore_missing: true - rename: + tag: rename_json_cvss_v3_confidentiality_impact_to_rapid7_insightvm_vulnerability_cvss_v3_confidentiality_impact_341514fc field: json.cvss_v3_confidentiality_impact target_field: rapid7.insightvm.vulnerability.cvss.v3.confidentiality_impact ignore_missing: true @@ -200,6 +231,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_f315eda5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -211,17 +243,21 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_3ced4255 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_cvss_v3_integrity_impact_to_rapid7_insightvm_vulnerability_cvss_v3_integrity_impact_6a434e56 field: json.cvss_v3_integrity_impact target_field: rapid7.insightvm.vulnerability.cvss.v3.integrity_impact ignore_missing: true - rename: + tag: rename_json_cvss_v3_privileges_required_to_rapid7_insightvm_vulnerability_cvss_v3_privileges_required_dd3341ac field: json.cvss_v3_privileges_required target_field: rapid7.insightvm.vulnerability.cvss.v3.privileges_required ignore_missing: true - rename: + tag: rename_json_cvss_v3_scope_to_rapid7_insightvm_vulnerability_cvss_v3_scope_1f0b86f4 field: json.cvss_v3_scope target_field: rapid7.insightvm.vulnerability.cvss.v3.scope ignore_missing: true @@ -234,14 +270,17 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_d7c54ff1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: + tag: append_vulnerability_score_base_99e55694 field: vulnerability.score.base value: '{{{rapid7.insightvm.vulnerability.cvss.v3.score}}}' if: ctx.rapid7?.insightvm?.vulnerability?.cvss.v3.score != null allow_duplicates: false - foreach: + tag: foreach_vulnerability_score_base_d9f33703 field: vulnerability.score.base if: ctx.vulnerability?.score?.base instanceof List processor: @@ -258,10 +297,12 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_cvss_v3_user_interaction_to_rapid7_insightvm_vulnerability_cvss_v3_user_interaction_359e1abe field: json.cvss_v3_user_interaction target_field: rapid7.insightvm.vulnerability.cvss.v3.user_interaction ignore_missing: true - rename: + tag: rename_json_cvss_v3_vector_to_rapid7_insightvm_vulnerability_cvss_v3_vector_1d764cac field: json.cvss_v3_vector target_field: rapid7.insightvm.vulnerability.cvss.v3.vector ignore_missing: true @@ -274,33 +315,41 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_830fc387 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_description_to_rapid7_insightvm_vulnerability_description_2fc93726 field: json.description target_field: rapid7.insightvm.vulnerability.description ignore_missing: true - set: + tag: set_vulnerability_description_0ff564b3 field: vulnerability.description copy_from: rapid7.insightvm.vulnerability.description ignore_empty_value: true - rename: + tag: rename_json_exploits_to_rapid7_insightvm_vulnerability_exploits_3a397a3c field: json.exploits target_field: rapid7.insightvm.vulnerability.exploits ignore_missing: true - rename: + tag: rename_json_id_to_rapid7_insightvm_vulnerability_id_964ed5a2 field: json.id target_field: rapid7.insightvm.vulnerability.id ignore_missing: true - set: + tag: set_event_id_5d222f6b field: event.id copy_from: rapid7.insightvm.vulnerability.id ignore_empty_value: true - rename: + tag: rename_json_links_to_rapid7_insightvm_vulnerability_links_b3a2afb6 field: json.links target_field: rapid7.insightvm.vulnerability.links ignore_missing: true - rename: + tag: rename_json_malware_kits_to_rapid7_insightvm_vulnerability_malware_kits_8e204d42 field: json.malware_kits target_field: rapid7.insightvm.vulnerability.malware_kits ignore_missing: true @@ -313,9 +362,11 @@ processors: if: ctx.json?.modified != null && ctx.json?.modified != '' on_failure: - append: + tag: append_error_message_13e7b20d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_timestamp_75687ba8 field: '@timestamp' copy_from: rapid7.insightvm.vulnerability.modified ignore_empty_value: true @@ -328,6 +379,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4ff8e70a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -339,6 +391,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8e8e55d0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -350,13 +403,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_e608fc0c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_pci_special_notes_to_rapid7_insightvm_vulnerability_pci_special_notes_3135bd15 field: json.pci_special_notes target_field: rapid7.insightvm.vulnerability.pci.special_notes ignore_missing: true - rename: + tag: rename_json_pci_status_to_rapid7_insightvm_vulnerability_pci_status_f29c30bf field: json.pci_status target_field: rapid7.insightvm.vulnerability.pci.status ignore_missing: true @@ -369,13 +425,16 @@ processors: if: ctx.json?.published != null && ctx.json?.published != '' on_failure: - append: + tag: append_error_message_9924d478 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_references_to_rapid7_insightvm_vulnerability_references_8f281818 field: json.references target_field: rapid7.insightvm.vulnerability.references ignore_missing: true - set: + tag: set_vulnerability_reference_17775668 field: vulnerability.reference copy_from: rapid7.insightvm.vulnerability.references ignore_empty_value: true @@ -388,23 +447,28 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_79f7c71d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_event_risk_score_fca0d4c1 field: event.risk_score copy_from: rapid7.insightvm.vulnerability.risk_score ignore_empty_value: true - script: + tag: script_32083996 description: Normalize event.risk_score to event.risk_score_norm lang: painless if: ctx.event?.risk_score != null ignore_failure: true source: def normalizedRiskScore = ctx.event.risk_score / 10.0; ctx.event.risk_score_norm = normalizedRiskScore; - rename: + tag: rename_json_severity_to_rapid7_insightvm_vulnerability_severity_679ad872 field: json.severity target_field: rapid7.insightvm.vulnerability.severity ignore_missing: true - set: + tag: set_vulnerability_severity_3cfdcd3d field: vulnerability.severity copy_from: rapid7.insightvm.vulnerability.severity ignore_empty_value: true @@ -417,20 +481,25 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9eab40c7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_event_severity_75933a6a field: event.severity copy_from: rapid7.insightvm.vulnerability.severity_score ignore_empty_value: true - rename: + tag: rename_json_title_to_rapid7_insightvm_vulnerability_title_a150e1e6 field: json.title target_field: rapid7.insightvm.vulnerability.title ignore_missing: true - remove: + tag: remove_json_29cdffdc field: json ignore_missing: true - remove: + tag: remove_be7a804b if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - rapid7.insightvm.vulnerability.modified @@ -445,6 +514,7 @@ processors: - rapid7.insightvm.vulnerability.severity ignore_missing: true - script: + tag: script_06971239 lang: painless description: Drops null/empty values recursively. source: |- @@ -462,10 +532,12 @@ processors: } drop(ctx); - set: + tag: set_event_kind_92954dfa field: event.kind value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/rapid7_insightvm/manifest.yml b/packages/rapid7_insightvm/manifest.yml index ef3a4eb38d3..32da473c5d1 100644 --- a/packages/rapid7_insightvm/manifest.yml +++ b/packages/rapid7_insightvm/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.4.0" name: rapid7_insightvm title: Rapid7 InsightVM -version: "2.9.2" +version: "2.9.3" source: license: "Elastic-2.0" description: Collect logs from Rapid7 InsightVM with Elastic Agent. diff --git a/packages/sailpoint_identity_sc/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/sailpoint_identity_sc/data_stream/events/elasticsearch/ingest_pipeline/default.yml index f752f55d850..1b2b715e64a 100644 --- a/packages/sailpoint_identity_sc/data_stream/events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/sailpoint_identity_sc/data_stream/events/elasticsearch/ingest_pipeline/default.yml @@ -2,6 +2,7 @@ description: Pipeline for processing sailpoint identity security cloud events processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: "8.11.0" - terminate: @@ -10,6 +11,7 @@ processors: description: error message set and no data to process. - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -21,6 +23,7 @@ processors: if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original_to_sailpoint_identity_sc_events_a30ddfa6 field: event.original target_field: sailpoint_identity_sc.events @@ -31,10 +34,12 @@ processors: target_field: "_id" on_failure: - append: + tag: append_error_message_e87bbabd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' -##################### Scripts for processing input ##################### + ##################### Scripts for processing input ##################### - script: + tag: script_a3eb5add description: Drops null/empty values recursively. lang: painless source: | @@ -52,6 +57,7 @@ processors: } dropEmptyFields(ctx); + - script: description: Convert Additional Info keys to snake case. tag: additional-info-keys-to-snake-case @@ -81,86 +87,102 @@ processors: if (ctx.sailpoint_identity_sc.events != null) { ctx.sailpoint_identity_sc.events = keysToSnakeCase(ctx.sailpoint_identity_sc.events); } -##################### ECS Fields ##################### + ##################### ECS Fields ##################### - set: + tag: set_event_kind_de80643c field: event.kind value: event - append: + tag: append_event_category_576de059 field: event.category value: iam - set: + tag: set_event_dataset_ae8e1e76 field: event.dataset value: sailpoint_identity_sc.events - set: + tag: set_event_module_0f43d717 field: event.module value: sailpoint_identity_sc - set: + tag: set_event_type_ec95f7f2 field: event.type value: ["info"] - set: + tag: set_event_dataset_ae8e1e76 field: event.dataset value: sailpoint_identity_sc.events - set: + tag: set_event_module_0f43d717 field: event.module value: sailpoint_identity_sc - set: + tag: set_timestamp_173de85f field: "@timestamp" copy_from: sailpoint_identity_sc.events.created ignore_empty_value: true - set: + tag: set_event_id_dcd98e5a field: event.id copy_from: ctx.sailpoint_identity_sc.events.id ignore_empty_value: true - append: + tag: append_related_hosts_fdfad510 field: related.hosts value: "{{{sailpoint_identity_sc.events.ip_address}}}" allow_duplicates: false if: ctx.sailpoint_identity_sc.events?.ip_address != null && ctx.sailpoint_identity_sc.events.ip_address != '' - + - append: + tag: append_related_user_cdb190ea field: related.user value: "{{{sailpoint_identity_sc.events.actor.name}}}" - allow_duplicates: false - if: ctx.sailpoint_identity_sc.events?.actor.name != null && ctx.sailpoint_identity_sc.events.actor.name != '' + allow_duplicates: false + if: ctx.sailpoint_identity_sc.events?.actor.name != null && ctx.sailpoint_identity_sc.events.actor.name != '' - append: + tag: append_related_hosts_bfe467e9 field: related.hosts value: "{{{sailpoint_identity_sc.events.attributes.host_name}}}" - allow_duplicates: false - if: ctx.sailpoint_identity_sc.events?.host_name != null && ctx.sailpoint_identity_sc.events.host_name != '' + allow_duplicates: false + if: ctx.sailpoint_identity_sc.events?.host_name != null && ctx.sailpoint_identity_sc.events.host_name != '' - #################### Override host.ip ###################### + #################### Override host.ip ###################### - convert: + tag: convert_sailpoint_identity_sc_events_ip_address_5c4563e7 field: sailpoint_identity_sc.events.ip_address type: ip if: ctx.sailpoint_identity_sc.events?.ip_address != null && ctx.sailpoint_identity_sc.events.ip_address != '' ignore_missing: true - remove: + tag: remove_sailpoint_identity_sc_events_ip_address_ab417715 field: sailpoint_identity_sc.events.ip_address - if: ctx.sailpoint_identity_sc.events?.ip_address == "" + if: ctx.sailpoint_identity_sc.events?.ip_address == "" ignore_missing: true - append: + tag: append_host_ip_4930e50f field: host.ip - value: "{{{sailpoint_identity_sc.events.ip_address}}}" - allow_duplicates: false + value: "{{{sailpoint_identity_sc.events.ip_address}}}" + allow_duplicates: false if: ctx.sailpoint_identity_sc.events?.ip_address != null && ctx.sailpoint_identity_sc.events.ip_address != '' - geoip: + tag: geoip_host_ip_to_host_geo_b6f95445 field: host.ip target_field: host.geo ignore_missing: true ignore_failure: true - ################### Entity fields ################### + ################### Entity fields ################### - set: field: user.entity.id tag: set_user_entity_id @@ -192,13 +214,14 @@ processors: tag: set_user_entity_lifecycle_last_activity copy_from: '@timestamp' if: ctx.sailpoint_identity_sc?.events?.actor?.id != null || ctx.sailpoint_identity_sc?.events?.actor?.name != null - ################### Error Log fields ################### + ################### Error Log fields ################### - set: field: event.kind tag: set_pipeline_error_into_event_kind value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/santa/changelog.yml b/packages/santa/changelog.yml index 74d913baba3..92065bd810b 100644 --- a/packages/santa/changelog.yml +++ b/packages/santa/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.24.1" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "3.24.0" changes: - description: Update Kibana constraint to support 9.0.0. diff --git a/packages/santa/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/santa/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 86f70fca73d..2bddbf332d7 100644 --- a/packages/santa/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/santa/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -2,14 +2,17 @@ description: Pipeline for parsing Google Santa logs. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null - grok: + tag: grok_event_original_f6f5a506 field: event.original patterns: - '\[%{TIMESTAMP_ISO8601:_tmp.timestamp}\] %{NOT_SEPARATOR:log.level} santad: %{GREEDYDATA:_tmp.message}' @@ -25,153 +28,190 @@ processors: ignore_missing: true on_failure: - fail: + tag: fail_52c71a6a message: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" - remove: + tag: remove__tmp_message_4276d2ef field: _tmp.message ignore_missing: true ## ECS fields - rename: + tag: rename_santa_gid_to_group_id_98c7d37d field: santa.gid target_field: group.id ignore_missing: true - rename: + tag: rename_santa_group_to_group_name_e8e7d8e2 field: santa.group target_field: group.name ignore_missing: true - rename: + tag: rename_santa_newpath_to_file_target_path_c1b6f553 field: santa.newpath target_field: file.target_path ignore_missing: true - rename: + tag: rename_santa_path_to_file_path_b7335bd7 field: santa.path target_field: file.path ignore_missing: true - rename: + tag: rename_santa_pid_to_process_pid_58e09f22 field: santa.pid target_field: process.pid ignore_missing: true - convert: + tag: convert_process_pid_df04a3d5 field: process.pid type: long ignore_missing: true - rename: + tag: rename_santa_ppid_to_process_parent_pid_c6a2b8de field: santa.ppid target_field: process.parent.pid ignore_missing: true - convert: + tag: convert_process_parent_pid_1315c9e1 field: process.parent.pid type: long ignore_missing: true - rename: + tag: rename_santa_process_to_process_name_7ec19b4e field: santa.process target_field: process.name ignore_missing: true - rename: + tag: rename_santa_processpath_to_process_executable_de83c85e field: santa.processpath target_field: process.executable ignore_missing: true - rename: + tag: rename_santa_sha256_to_process_hash_sha256_65933fdc field: santa.sha256 target_field: process.hash.sha256 ignore_missing: true - rename: + tag: rename_santa_uid_to_user_id_f2ceb8bd field: santa.uid target_field: user.id ignore_missing: true - rename: + tag: rename_santa_user_to_user_name_c9102812 field: santa.user target_field: user.name ignore_missing: true ## Santa custom fields - rename: + tag: rename_santa_cert_sha256_to_santa_certificate_sha256_95973d50 field: santa.cert_sha256 target_field: santa.certificate.sha256 ignore_missing: true - rename: + tag: rename_santa_cert_cn_to_santa_certificate_common_name_3cdd4252 field: santa.cert_cn target_field: santa.certificate.common_name ignore_missing: true - rename: + tag: rename_santa_event_uid_to_santa_event_uid_29dbfbf9 field: santa.event_uid target_field: santa.event.uid ignore_missing: true - rename: + tag: rename_santa_event_user_to_santa_event_user_432785d9 field: santa.event_user target_field: santa.event.user ignore_missing: true - rename: + tag: rename_santa_appearance_to_santa_disk_appearance_b42b0347 field: santa.appearance target_field: santa.disk.appearance ignore_missing: true - rename: + tag: rename_santa_bsdname_to_santa_disk_bsdname_65e1cd2d field: santa.bsdname target_field: santa.disk.bsdname ignore_missing: true - rename: + tag: rename_santa_bus_to_santa_disk_bus_9981fc3d field: santa.bus target_field: santa.disk.bus ignore_missing: true - rename: + tag: rename_santa_dmgpath_to_santa_disk_dmgpath_7f48fe15 field: santa.dmgpath target_field: santa.disk.dmgpath ignore_missing: true - rename: + tag: rename_santa_fs_to_santa_disk_fs_e02acb15 field: santa.fs target_field: santa.disk.fs ignore_missing: true - convert: + tag: convert_santa_graphical_session_id_8a6776ee field: santa.graphical_session_id type: long ignore_missing: true - rename: + tag: rename_santa_model_to_santa_disk_model_cb225475 field: santa.model target_field: santa.disk.model ignore_missing: true - rename: + tag: rename_santa_mount_to_santa_disk_mount_9c811b61 field: santa.mount target_field: santa.disk.mount ignore_missing: true - convert: + tag: convert_santa_pidversion_2117ab6d field: santa.pidversion type: long ignore_missing: true - rename: + tag: rename_santa_serial_to_santa_disk_serial_61bd4313 field: santa.serial target_field: santa.disk.serial ignore_missing: true - rename: + tag: rename_santa_volume_to_santa_disk_volume_9e32ad4b field: santa.volume target_field: santa.disk.volume ignore_missing: true - rename: + tag: rename_santa_teamid_to_santa_team_id_ecb29e75 field: santa.teamid target_field: santa.team_id ignore_missing: true - set: + tag: set_process_entity_id_1d9ac266 field: process.entity_id value: "{{{process.pid}}}-{{{santa.pidversion}}}" if: "ctx.process?.pid != null && ctx.santa?.pidversion != null" - set: + tag: set_process_entity_id_67f36b6e field: process.entity_id value: "{{{agent.id}}}-{{{process.entity_id}}}" if: "ctx.agent?.id != null && ctx.process?.entity_id != null" - date: + tag: date__tmp_timestamp_54909160 field: _tmp.timestamp formats: - ISO8601 ignore_failure: true - remove: + tag: remove__tmp_timestamp_7a88ba46 field: _tmp.timestamp ignore_missing: true - set: + tag: set_process_start_4dee873c field: process.start copy_from: "@timestamp" if: ctx.process?.pid != null ignore_empty_value: true - date: + tag: date_santa_disk_appearance_to_santa_disk_appearance_046340cf field: santa.disk.appearance target_field: santa.disk.appearance formats: @@ -179,19 +219,23 @@ processors: ignore_failure: true if: ctx.santa?.disk?.appearance != null - split: + tag: split_santa_args_fc2dcd07 field: santa.args separator: ' ' ignore_failure: true - rename: + tag: rename_file_path_to_process_executable_ac6ee5c0 field: file.path target_field: process.executable if: ctx.process?.pid != null && ctx.process?.executable == null ignore_missing: true - append: + tag: append_process_args_13223491 field: process.args value: "{{{process.executable}}}" if: "ctx?.process?.executable != null" - foreach: + tag: foreach_santa_args_72fd5234 field: santa.args processor: append: @@ -199,52 +243,65 @@ processors: value: "{{{_ingest._value}}}" ignore_missing: true - remove: + tag: remove_santa_args_ff075e8a field: santa.args ignore_missing: true - set: + tag: set_event_kind_de80643c field: event.kind value: event - append: + tag: append_event_category_a8732667 field: event.category value: process if: "ctx?.santa?.action == 'EXEC'" - append: + tag: append_event_type_8789cd44 field: event.type value: start if: "ctx?.santa?.action == 'EXEC'" - set: + tag: set_event_outcome_757077c5 field: event.outcome value: success if: "ctx?.santa?.decision == 'ALLOW'" - set: + tag: set_event_outcome_397d48e3 field: event.outcome value: failure if: "ctx?.santa?.decision == 'DENY'" - set: + tag: set_event_action_8aa9c9f3 field: event.action value: "{{{santa.action}}}" ignore_empty_value: true - lowercase: + tag: lowercase_event_action_9334b869 field: event.action ignore_missing: true - append: + tag: append_related_user_6ab25889 field: related.user value: "{{{user.name}}}" if: "ctx?.user?.name != null" - append: + tag: append_related_hash_8bc614a5 field: related.hash value: "{{{santa.certificate.sha256}}}" if: "ctx?.santa?.certificate?.sha256 != null" - append: + tag: append_related_hash_b93d0a35 field: related.hash value: "{{{process.hash.sha256}}}" if: "ctx?.process?.hash != null" - append: + tag: append_file_x509_issuer_common_name_06d3ce2f field: file.x509.issuer.common_name value: "{{{santa.certificate.common_name}}}" if: ctx.santa?.certificate?.common_name != null - script: + tag: script_9d6fa8ba description: Drops null/empty values recursively lang: painless ignore_failure: true diff --git a/packages/santa/manifest.yml b/packages/santa/manifest.yml index 5ae408ce748..7de79015dc2 100644 --- a/packages/santa/manifest.yml +++ b/packages/santa/manifest.yml @@ -1,6 +1,6 @@ name: santa title: Google Santa -version: "3.24.0" +version: "3.24.1" description: Collect logs from Google Santa with Elastic Agent. type: integration icons: diff --git a/packages/sentinel_one/changelog.yml b/packages/sentinel_one/changelog.yml index 90ea9a764c1..e13d65b20cb 100644 --- a/packages/sentinel_one/changelog.yml +++ b/packages/sentinel_one/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.11.4" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "2.11.3" changes: - description: Fix error handling for first phase of agent data collection. diff --git a/packages/sentinel_one/data_stream/unified_alert/elasticsearch/ingest_pipeline/default.yml b/packages/sentinel_one/data_stream/unified_alert/elasticsearch/ingest_pipeline/default.yml index cfd76385580..4c8b3bb1c7d 100644 --- a/packages/sentinel_one/data_stream/unified_alert/elasticsearch/ingest_pipeline/default.yml +++ b/packages/sentinel_one/data_stream/unified_alert/elasticsearch/ingest_pipeline/default.yml @@ -23,7 +23,6 @@ processors: description: >- Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - # parse the event JSON - rename: field: message @@ -141,9 +140,11 @@ processors: if: ctx.sentinel_one?.unified_alert?.created_at != null && ctx.sentinel_one.unified_alert.created_at != '' on_failure: - remove: + tag: remove_sentinel_one_unified_alert_created_at_16e72ae4 field: sentinel_one.unified_alert.created_at ignore_missing: true - append: + tag: append_error_message_f52c0153 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -206,9 +207,11 @@ processors: if: ctx.sentinel_one?.unified_alert?.detected_at != null && ctx.sentinel_one.unified_alert.detected_at != '' on_failure: - remove: + tag: remove_sentinel_one_unified_alert_detected_at_4fb62f86 field: sentinel_one.unified_alert.detected_at ignore_missing: true - append: + tag: append_error_message_19778309 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: @@ -302,8 +305,10 @@ processors: if: ctx.sentinel_one?.unified_alert?.detection_time?.attacker?.ip != null && ctx.sentinel_one.unified_alert.detection_time.attacker.ip != '' on_failure: - remove: + tag: remove_sentinel_one_unified_alert_detection_time_attacker_ip_19ef4804 field: sentinel_one.unified_alert.detection_time.attacker.ip - append: + tag: append_error_message_b67c853d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -315,9 +320,11 @@ processors: if: ctx.sentinel_one?.unified_alert?.first_seen_at != null && ctx.sentinel_one.unified_alert.first_seen_at != '' on_failure: - remove: + tag: remove_sentinel_one_unified_alert_first_seen_at_9adab504 field: sentinel_one.unified_alert.first_seen_at ignore_missing: true - append: + tag: append_error_message_5d290641 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -329,9 +336,11 @@ processors: if: ctx.sentinel_one?.unified_alert?.last_seen_at != null && ctx.sentinel_one.unified_alert.last_seen_at != '' on_failure: - remove: + tag: remove_sentinel_one_unified_alert_last_seen_at_e863d95e field: sentinel_one.unified_alert.last_seen_at ignore_missing: true - append: + tag: append_error_message_be8332a7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -341,8 +350,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_sentinel_one_unified_alert_note_exists_f58e43c6 field: sentinel_one.unified_alert.note_exists - append: + tag: append_error_message_e4fc773f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -352,8 +363,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_sentinel_one_unified_alert_slo_details_time_to_resolve_data_action_complete_82358640 field: sentinel_one.unified_alert.slo_details.time_to_resolve_data.action_complete - append: + tag: append_error_message_adb3a036 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -363,8 +376,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_sentinel_one_unified_alert_slo_details_time_to_resolve_data_action_due_2ce0eda8 field: sentinel_one.unified_alert.slo_details.time_to_resolve_data.action_due - append: + tag: append_error_message_fc2d1b75 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -374,8 +389,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_sentinel_one_unified_alert_slo_details_time_to_resolve_data_completion_6984c4d0 field: sentinel_one.unified_alert.slo_details.time_to_resolve_data.completion - append: + tag: append_error_message_5bc67722 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -387,9 +404,11 @@ processors: if: ctx.sentinel_one?.unified_alert?.slo_details?.time_to_resolve_data?.completion_time != null && ctx.sentinel_one.unified_alert.slo_details.time_to_resolve_data.completion_time != '' on_failure: - remove: + tag: remove_sentinel_one_unified_alert_slo_details_time_to_resolve_data_completion_time_816484dc field: sentinel_one.unified_alert.slo_details.time_to_resolve_data.completion_time ignore_missing: true - append: + tag: append_error_message_82581e5f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -399,8 +418,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_sentinel_one_unified_alert_slo_details_time_to_resolve_data_target_6440b690 field: sentinel_one.unified_alert.slo_details.time_to_resolve_data.target - append: + tag: append_error_message_f93c684f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -412,9 +433,11 @@ processors: if: ctx.sentinel_one?.unified_alert?.slo_details?.time_to_resolve_data?.target_time != null && ctx.sentinel_one.unified_alert.slo_details.time_to_resolve_data.target_time != '' on_failure: - remove: + tag: remove_sentinel_one_unified_alert_slo_details_time_to_resolve_data_target_time_3004a841 field: sentinel_one.unified_alert.slo_details.time_to_resolve_data.target_time ignore_missing: true - append: + tag: append_error_message_ab1646d5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -424,8 +447,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_sentinel_one_unified_alert_slo_details_time_to_response_data_action_complete_6991e8fe field: sentinel_one.unified_alert.slo_details.time_to_response_data.action_complete - append: + tag: append_error_message_f1c5a08f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -435,8 +460,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_sentinel_one_unified_alert_slo_details_time_to_response_data_action_due_50a1250a field: sentinel_one.unified_alert.slo_details.time_to_response_data.action_due - append: + tag: append_error_message_0d472e7e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -446,8 +473,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_sentinel_one_unified_alert_slo_details_time_to_response_data_completion_863bedce field: sentinel_one.unified_alert.slo_details.time_to_response_data.completion - append: + tag: append_error_message_b38fd69f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -459,9 +488,11 @@ processors: if: ctx.sentinel_one?.unified_alert?.slo_details?.time_to_response_data?.completion_time != null && ctx.sentinel_one.unified_alert.slo_details.time_to_response_data.completion_time != '' on_failure: - remove: + tag: remove_sentinel_one_unified_alert_slo_details_time_to_response_data_completion_time_b19b1123 field: sentinel_one.unified_alert.slo_details.time_to_response_data.completion_time ignore_missing: true - append: + tag: append_error_message_6ab83917 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -471,8 +502,10 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_sentinel_one_unified_alert_slo_details_time_to_response_data_target_0ffdfc92 field: sentinel_one.unified_alert.slo_details.time_to_response_data.target - append: + tag: append_error_message_296279f0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -484,9 +517,11 @@ processors: if: ctx.sentinel_one?.unified_alert?.slo_details?.time_to_response_data?.target_time != null && ctx.sentinel_one.unified_alert.slo_details.time_to_response_data.target_time != '' on_failure: - remove: + tag: remove_sentinel_one_unified_alert_slo_details_time_to_response_data_target_time_4c9c0bf8 field: sentinel_one.unified_alert.slo_details.time_to_response_data.target_time ignore_missing: true - append: + tag: append_error_message_a79bc19f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -498,9 +533,11 @@ processors: if: ctx.sentinel_one?.unified_alert?.updated_at != null && ctx.sentinel_one.unified_alert.updated_at != '' on_failure: - remove: + tag: remove_sentinel_one_unified_alert_updated_at_f1ad362d field: sentinel_one.unified_alert.updated_at ignore_missing: true - append: + tag: append_error_message_2faef1e5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' @@ -589,6 +626,7 @@ processors: } on_failure: - append: + tag: append_error_message_8d328ad0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -752,6 +790,7 @@ processors: } on_failure: - append: + tag: append_error_message_cd4684dd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' @@ -766,7 +805,7 @@ processors: tag: set_process_parent_name_from_unified_alert_process_parent_name copy_from: sentinel_one.unified_alert.process.parent_name ignore_empty_value: true - + # file.* - set: field: file.hash.md5 @@ -793,7 +832,7 @@ processors: tag: set_file_hash_sha256_from_unified_alert_process_file_sha256 copy_from: sentinel_one.unified_alert.process.file.sha256 ignore_empty_value: true - + # related.* - append: field: related.hosts @@ -965,6 +1004,7 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/sentinel_one/manifest.yml b/packages/sentinel_one/manifest.yml index 9006b00db81..c6fd89b9a25 100644 --- a/packages/sentinel_one/manifest.yml +++ b/packages/sentinel_one/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.4.0" name: sentinel_one title: SentinelOne -version: "2.11.3" +version: "2.11.4" description: Collect logs from SentinelOne with Elastic Agent. type: integration categories: diff --git a/packages/sentinel_one_cloud_funnel/changelog.yml b/packages/sentinel_one_cloud_funnel/changelog.yml index ff2cf306784..f1525580954 100644 --- a/packages/sentinel_one_cloud_funnel/changelog.yml +++ b/packages/sentinel_one_cloud_funnel/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.13.2" + changes: + - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. + type: enhancement + link: https://github.com/elastic/integrations/issues/20558 - version: "1.13.1" changes: - description: Remove duplicated installation instructions from the documentation diff --git a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/default.yml index 2f1b66d8db1..b5348344bb6 100644 --- a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -2,21 +2,26 @@ description: Pipeline for processing Event logs. processors: - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: 8.11.0 - set: + tag: set_event_kind_de80643c field: event.kind value: event - gsub: + tag: gsub_message_9e16a0bd field: message pattern: '\\\\\\\\' replacement: '\\\\' ignore_missing: true on_failure: - append: + tag: append_error_message_b5b074f8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_message_to_event_original_56a77271 field: message target_field: event.original ignore_missing: true @@ -27,6 +32,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message_a6e0e953 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -39,6 +45,7 @@ processors: ctx.json.remove('k8sCluster.containerImage'); on_failure: - append: + tag: append_error_message_2f5f3f4d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - dot_expander: @@ -47,6 +54,7 @@ processors: path: json on_failure: - append: + tag: append_error_message_8702f43e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -74,6 +82,7 @@ processors: double_quote: "\"" on_failure: - append: + tag: append_error_message_7050d87d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -86,6 +95,7 @@ processors: if: ctx.json?.event?.time != null && ctx.json.event.time != '' on_failure: - append: + tag: append_error_message_86f53aaf field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -97,109 +107,135 @@ processors: if: ctx.json?.event?.time != null && ctx.json.event.time != '' on_failure: - append: + tag: append_error_message_1aa3e3c3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_event_logout_tgt_user_name_to_sentinel_one_cloud_funnel_event_logout_tgt_user_name_aebc319e field: json.event.logout.tgt.user.name target_field: sentinel_one_cloud_funnel.event.logout.tgt.user.name ignore_missing: true - set: + tag: set_destination_user_name_b7e906fe field: destination.user.name copy_from: sentinel_one_cloud_funnel.event.logout.tgt.user.name ignore_empty_value: true - rename: + tag: rename_json_event_login_tgt_domainName_to_sentinel_one_cloud_funnel_event_login_tgt_domain_name_67294137 field: json.event.login.tgt.domainName target_field: sentinel_one_cloud_funnel.event.login.tgt.domain_name ignore_missing: true - set: + tag: set_destination_user_domain_5b45dfba field: destination.user.domain copy_from: sentinel_one_cloud_funnel.event.login.tgt.domain_name ignore_empty_value: true - rename: + tag: rename_json_event_logout_tgt_domainName_to_sentinel_one_cloud_funnel_event_logout_tgt_domain_name_883c0f09 field: json.event.logout.tgt.domainName target_field: sentinel_one_cloud_funnel.event.logout.tgt.domain_name ignore_missing: true - set: + tag: set_destination_user_domain_c9f05585 field: destination.user.domain copy_from: sentinel_one_cloud_funnel.event.logout.tgt.domain_name ignore_empty_value: true - rename: + tag: rename_json_event_login_tgt_user_name_to_sentinel_one_cloud_funnel_event_login_tgt_user_name_36f0d258 field: json.event.login.tgt.user.name target_field: sentinel_one_cloud_funnel.event.login.tgt.user.name ignore_missing: true - set: + tag: set_destination_user_name_741dcc2d field: destination.user.name copy_from: sentinel_one_cloud_funnel.event.login.tgt.user.name ignore_empty_value: true - append: + tag: append_related_user_8f37bc82 field: related.user value: '{{{destination.user.name}}}' allow_duplicates: false if: ctx.destination?.user?.name != null - append: + tag: append_related_user_edf75b08 field: related.user value: '{{{destination.user.domain}}}' allow_duplicates: false if: ctx.destination?.user?.domain != null - rename: + tag: rename_json_event_id_to_sentinel_one_cloud_funnel_event_id_49f27aee field: json.event.id target_field: sentinel_one_cloud_funnel.event.id ignore_missing: true - set: + tag: set_event_id_f088bfd1 field: event.id copy_from: sentinel_one_cloud_funnel.event.id ignore_empty_value: true - rename: + tag: rename_json_tgt_file_name_to_sentinel_one_cloud_funnel_event_tgt_file_name_76a95ef2 field: json.tgt.file.name target_field: sentinel_one_cloud_funnel.event.tgt.file.name ignore_missing: true - set: + tag: set_file_name_f3025986 field: file.name copy_from: sentinel_one_cloud_funnel.event.tgt.file.name ignore_empty_value: true - rename: + tag: rename_json_group_id_to_sentinel_one_cloud_funnel_event_group_id_1ee2530c field: json.group.id target_field: sentinel_one_cloud_funnel.event.group.id ignore_missing: true - set: + tag: set_group_id_766b0a15 field: group.id copy_from: sentinel_one_cloud_funnel.event.group.id ignore_empty_value: true - rename: + tag: rename_json_endpoint_name_to_sentinel_one_cloud_funnel_event_endpoint_name_f4761db6 field: json.endpoint.name target_field: sentinel_one_cloud_funnel.event.endpoint.name ignore_missing: true - set: + tag: set_host_hostname_c6e353d8 field: host.hostname copy_from: sentinel_one_cloud_funnel.event.endpoint.name ignore_empty_value: true - append: + tag: append_related_hosts_7117c4ab field: related.hosts value: '{{{sentinel_one_cloud_funnel.event.endpoint.name}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.endpoint?.name != null - rename: + tag: rename_json_os_name_to_sentinel_one_cloud_funnel_event_os_name_e8764d45 field: json.os.name target_field: sentinel_one_cloud_funnel.event.os_name ignore_missing: true - set: + tag: set_host_os_name_2c4279a2 field: host.os.name copy_from: sentinel_one_cloud_funnel.event.os_name ignore_empty_value: true - append: + tag: append_related_hosts_4a2794c4 field: related.hosts value: '{{{sentinel_one_cloud_funnel.event.os_name}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.os?.name != null - rename: + tag: rename_json_endpoint_os_to_sentinel_one_cloud_funnel_event_endpoint_os_e533f6fc field: json.endpoint.os target_field: sentinel_one_cloud_funnel.event.endpoint.os ignore_missing: true - set: + tag: set_host_os_platform_51c08359 field: host.os.platform copy_from: sentinel_one_cloud_funnel.event.endpoint.os ignore_empty_value: true - set: + tag: set_host_os_type_dfa90718 field: host.os.type copy_from: sentinel_one_cloud_funnel.event.endpoint.os if: >- @@ -211,142 +247,175 @@ processors: 'android' ].contains(ctx.sentinel_one_cloud_funnel?.event?.endpoint?.os) - append: + tag: append_related_hosts_83ecc0d5 field: related.hosts value: '{{{sentinel_one_cloud_funnel.event.endpoint.os}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.endpoint?.os != null - rename: + tag: rename_json_endpoint_type_to_sentinel_one_cloud_funnel_event_endpoint_type_a0e5e900 field: json.endpoint.type target_field: sentinel_one_cloud_funnel.event.endpoint.type ignore_missing: true - set: + tag: set_host_type_9931fba6 field: host.type copy_from: sentinel_one_cloud_funnel.event.endpoint.type ignore_empty_value: true - append: + tag: append_related_hosts_65db4409 field: related.hosts value: '{{{sentinel_one_cloud_funnel.event.endpoint.type}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.endpoint?.type != null - rename: + tag: rename_json_src_process_cmdline_to_sentinel_one_cloud_funnel_event_src_process_cmd_line_dc98de1b field: json.src.process.cmdline target_field: sentinel_one_cloud_funnel.event.src.process.cmd_line ignore_missing: true - set: + tag: set_process_command_line_7c0f42d8 field: process.command_line copy_from: sentinel_one_cloud_funnel.event.src.process.cmd_line ignore_empty_value: true - rename: + tag: rename_json_src_process_image_path_to_sentinel_one_cloud_funnel_event_src_process_image_path_f55502a0 field: json.src.process.image.path target_field: sentinel_one_cloud_funnel.event.src.process.image.path ignore_missing: true - set: + tag: set_process_executable_fbba412d field: process.executable copy_from: sentinel_one_cloud_funnel.event.src.process.image.path ignore_empty_value: true - rename: + tag: rename_json_src_process_image_md5_to_sentinel_one_cloud_funnel_event_src_process_image_md5_efac9ac6 field: json.src.process.image.md5 target_field: sentinel_one_cloud_funnel.event.src.process.image.md5 ignore_missing: true - set: + tag: set_process_hash_md5_62bdc746 field: process.hash.md5 copy_from: sentinel_one_cloud_funnel.event.src.process.image.md5 ignore_empty_value: true - append: + tag: append_related_hash_e71b2329 field: related.hash value: '{{{process.hash.md5}}}' allow_duplicates: false if: ctx.process?.hash?.md5 != null - rename: + tag: rename_json_src_process_image_sha1_to_sentinel_one_cloud_funnel_event_src_process_image_sha1_ae2e1be4 field: json.src.process.image.sha1 target_field: sentinel_one_cloud_funnel.event.src.process.image.sha1 ignore_missing: true - set: + tag: set_process_hash_sha1_036722c8 field: process.hash.sha1 copy_from: sentinel_one_cloud_funnel.event.src.process.image.sha1 ignore_empty_value: true - append: + tag: append_related_hash_96df867f field: related.hash value: '{{{process.hash.sha1}}}' allow_duplicates: false if: ctx.process?.hash?.sha1 != null - rename: + tag: rename_json_src_process_image_sha256_to_sentinel_one_cloud_funnel_event_src_process_image_sha256_eae2f320 field: json.src.process.image.sha256 target_field: sentinel_one_cloud_funnel.event.src.process.image.sha256 ignore_missing: true - set: + tag: set_process_hash_sha256_d04ec2e0 field: process.hash.sha256 copy_from: sentinel_one_cloud_funnel.event.src.process.image.sha256 ignore_empty_value: true - append: + tag: append_related_hash_239dcc07 field: related.hash value: '{{{process.hash.sha256}}}' allow_duplicates: false if: ctx.process?.hash?.sha256 != null - rename: + tag: rename_json_src_process_name_to_sentinel_one_cloud_funnel_event_src_process_name_59a076d8 field: json.src.process.name target_field: sentinel_one_cloud_funnel.event.src.process.name ignore_missing: true - set: + tag: set_process_name_79888b19 field: process.name copy_from: sentinel_one_cloud_funnel.event.src.process.name ignore_empty_value: true - rename: + tag: rename_json_src_process_parent_cmdline_to_sentinel_one_cloud_funnel_event_src_process_parent_cmd_line_af3c5aff field: json.src.process.parent.cmdline target_field: sentinel_one_cloud_funnel.event.src.process.parent.cmd_line ignore_missing: true - set: + tag: set_process_parent_command_line_c479889c field: process.parent.command_line copy_from: sentinel_one_cloud_funnel.event.src.process.parent.cmd_line ignore_empty_value: true - rename: + tag: rename_json_osSrc_process_parent_image_path_to_sentinel_one_cloud_funnel_event_os_src_process_parent_image_path_85727fd4 field: json.osSrc.process.parent.image.path target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.image.path ignore_missing: true - rename: + tag: rename_json_osSrc_process_parent_image_md5_to_sentinel_one_cloud_funnel_event_os_src_process_parent_image_md5_61fe55e2 field: json.osSrc.process.parent.image.md5 target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.image.md5 ignore_missing: true - set: + tag: set_process_parent_hash_md5_fce0d50c field: process.parent.hash.md5 copy_from: sentinel_one_cloud_funnel.event.os_src_process.parent.image.md5 ignore_empty_value: true - append: + tag: append_related_hash_badcbdc2 field: related.hash value: '{{{process.parent.hash.md5}}}' allow_duplicates: false if: ctx.process?.parent?.hash?.md5 != null - rename: + tag: rename_json_src_process_parent_image_sha1_to_sentinel_one_cloud_funnel_event_src_process_parent_image_sha1_ef8f05a0 field: json.src.process.parent.image.sha1 target_field: sentinel_one_cloud_funnel.event.src.process.parent.image.sha1 ignore_missing: true - set: + tag: set_process_parent_hash_sha1_7224cbba field: process.parent.hash.sha1 copy_from: sentinel_one_cloud_funnel.event.src.process.parent.image.sha1 ignore_empty_value: true - append: + tag: append_related_hash_156429a2 field: related.hash value: '{{{process.parent.hash.sha1}}}' allow_duplicates: false if: ctx.process?.parent?.hash?.sha1 != null - rename: + tag: rename_json_src_process_parent_image_sha256_to_sentinel_one_cloud_funnel_event_src_process_parent_image_sha256_9bb74564 field: json.src.process.parent.image.sha256 target_field: sentinel_one_cloud_funnel.event.src.process.parent.image.sha256 ignore_missing: true - set: + tag: set_process_parent_hash_sha256_6d0c65ca field: process.parent.hash.sha256 copy_from: sentinel_one_cloud_funnel.event.src.process.parent.image.sha256 ignore_empty_value: true - append: + tag: append_related_hash_fd8e170a field: related.hash value: '{{{process.parent.hash.sha256}}}' allow_duplicates: false if: ctx.process?.parent?.hash?.sha256 != null - rename: + tag: rename_json_src_process_parent_name_to_sentinel_one_cloud_funnel_event_src_process_parent_name_351c0d3a field: json.src.process.parent.name target_field: sentinel_one_cloud_funnel.event.src.process.parent.name ignore_missing: true - set: + tag: set_process_parent_name_faf2b5ed field: process.parent.name copy_from: sentinel_one_cloud_funnel.event.src.process.parent.name ignore_empty_value: true @@ -359,9 +428,11 @@ processors: if: ctx.json?.src?.process?.parent?.pid != '' on_failure: - append: + tag: append_error_message_c2b8e219 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_process_parent_pid_ae677049 field: process.parent.pid copy_from: sentinel_one_cloud_funnel.event.src.process.parent.pid ignore_empty_value: true @@ -374,26 +445,32 @@ processors: if: ctx.json?.src?.process?.parent?.rUserUid != '' on_failure: - append: + tag: append_error_message_51e20803 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_process_parent_real_user_id_f65779a2 field: process.parent.real_user.id copy_from: sentinel_one_cloud_funnel.event.src.process.parent.r_user.uid ignore_empty_value: true - append: + tag: append_related_user_c9d74ecb field: related.user value: '{{{process.parent.real_user.id}}}' allow_duplicates: false if: ctx.process?.parent?.real_user?.id != null - rename: + tag: rename_json_src_process_parent_rUserName_to_sentinel_one_cloud_funnel_event_src_process_parent_r_user_name_716f78e9 field: json.src.process.parent.rUserName target_field: sentinel_one_cloud_funnel.event.src.process.parent.r_user.name ignore_missing: true - set: + tag: set_process_parent_real_user_name_7ae564db field: process.parent.real_user.name copy_from: sentinel_one_cloud_funnel.event.src.process.parent.r_user.name ignore_empty_value: true - append: + tag: append_related_user_420c0b03 field: related.user value: '{{{process.parent.real_user.name}}}' allow_duplicates: false @@ -408,17 +485,21 @@ processors: if: ctx.json?.src?.process?.parent?.startTime != null && ctx.json.src.process.parent.startTime != '' on_failure: - append: + tag: append_error_message_6f0d636d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_process_parent_start_01360443 field: process.parent.start copy_from: sentinel_one_cloud_funnel.event.src.process.parent.start_time ignore_empty_value: true - rename: + tag: rename_json_src_process_parent_displayName_to_sentinel_one_cloud_funnel_event_src_process_parent_display_name_ae8a9029 field: json.src.process.parent.displayName target_field: sentinel_one_cloud_funnel.event.src.process.parent.display_name ignore_missing: true - set: + tag: set_process_parent_title_42369759 field: process.parent.title copy_from: sentinel_one_cloud_funnel.event.src.process.parent.display_name ignore_empty_value: true @@ -431,34 +512,42 @@ processors: if: ctx.json?.src?.process?.parent?.eUserUid != '' on_failure: - append: + tag: append_error_message_da873315 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_process_parent_user_id_a204e136 field: process.parent.user.id copy_from: sentinel_one_cloud_funnel.event.src.process.parent.e_user.uid ignore_empty_value: true - append: + tag: append_related_user_2ec538c1 field: related.user value: '{{{process.parent.user.id}}}' allow_duplicates: false if: ctx.process?.parent?.user?.id != null - rename: + tag: rename_json_src_process_parent_eUserName_to_sentinel_one_cloud_funnel_event_src_process_parent_e_user_name_a2f166f5 field: json.src.process.parent.eUserName target_field: sentinel_one_cloud_funnel.event.src.process.parent.e_user.name ignore_missing: true - set: + tag: set_process_parent_user_name_785c2f4f field: process.parent.user.name copy_from: sentinel_one_cloud_funnel.event.src.process.parent.e_user.name ignore_empty_value: true - rename: + tag: rename_json_src_process_parent_user_to_sentinel_one_cloud_funnel_event_src_process_parent_user_name_0e62ecf1 field: json.src.process.parent.user target_field: sentinel_one_cloud_funnel.event.src.process.parent.user.name ignore_missing: true - set: + tag: set_process_parent_user_name_fc9cf78f field: process.parent.user.name copy_from: sentinel_one_cloud_funnel.event.src.process.parent.user.name ignore_empty_value: true - append: + tag: append_related_user_e212dfed field: related.user value: '{{{process.parent.user.name}}}' allow_duplicates: false @@ -472,9 +561,11 @@ processors: if: ctx.json?.src?.process?.pid != '' on_failure: - append: + tag: append_error_message_b3654d53 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_process_pid_78e452ed field: process.pid copy_from: sentinel_one_cloud_funnel.event.src.process.pid ignore_empty_value: true @@ -487,6 +578,7 @@ processors: if: ctx.json?.tgt?.process?.pid != '' on_failure: - append: + tag: append_error_message_1dd51f92 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -498,26 +590,32 @@ processors: if: ctx.json?.src?.process?.rUserUid != '' on_failure: - append: + tag: append_error_message_9b581fe7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_process_real_user_id_e27c2984 field: process.real_user.id copy_from: sentinel_one_cloud_funnel.event.src.process.r_user.uid ignore_empty_value: true - append: + tag: append_related_user_eb470bac field: related.user value: '{{{process.real_user.id}}}' allow_duplicates: false if: ctx.process?.real_user?.id != null - rename: + tag: rename_json_src_process_rUserName_to_sentinel_one_cloud_funnel_event_src_process_r_user_name_fc26ab95 field: json.src.process.rUserName target_field: sentinel_one_cloud_funnel.event.src.process.r_user.name ignore_missing: true - set: + tag: set_process_real_user_name_9f3b170f field: process.real_user.name copy_from: sentinel_one_cloud_funnel.event.src.process.r_user.name ignore_empty_value: true - append: + tag: append_related_user_3747b694 field: related.user value: '{{{process.real_user.name}}}' allow_duplicates: false @@ -532,17 +630,21 @@ processors: if: ctx.json?.src?.process?.startTime != null && ctx.json.src.process.startTime != '' on_failure: - append: + tag: append_error_message_359e1c90 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_process_start_19fc6e65 field: process.start copy_from: sentinel_one_cloud_funnel.event.src.process.start_time ignore_empty_value: true - rename: + tag: rename_json_src_process_displayName_to_sentinel_one_cloud_funnel_event_src_process_display_name_523145c7 field: json.src.process.displayName target_field: sentinel_one_cloud_funnel.event.src.process.display_name ignore_missing: true - set: + tag: set_process_title_3c22e4cb field: process.title copy_from: sentinel_one_cloud_funnel.event.src.process.display_name ignore_empty_value: true @@ -555,87 +657,108 @@ processors: if: ctx.json?.src?.process?.eUserUid != '' on_failure: - append: + tag: append_error_message_e2526033 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_process_user_id_4990e1a6 field: process.user.id copy_from: sentinel_one_cloud_funnel.event.src.process.e_user.uid ignore_empty_value: true - rename: + tag: rename_json_src_process_user_to_sentinel_one_cloud_funnel_event_src_process_user_name_3763cb43 field: json.src.process.user target_field: sentinel_one_cloud_funnel.event.src.process.user.name ignore_missing: true - set: + tag: set_process_user_name_c5438569 field: process.user.name copy_from: sentinel_one_cloud_funnel.event.src.process.user.name ignore_empty_value: true - set: + tag: set_user_name_aced8732 field: user.name copy_from: process.user.name ignore_empty_value: true - set: + tag: set_user_domain_cbe01d27 field: user.domain copy_from: process.user.name ignore_empty_value: true - dissect: + tag: dissect_process_user_name_4225fe10 field: process.user.name pattern: '%{user.domain}\%{user.name}' if: ctx.process?.user?.name?.contains('\\') == true description: "Split user and domain" - rename: + tag: rename_json_src_process_eUserName_to_sentinel_one_cloud_funnel_event_src_process_e_user_name_728bc313 field: json.src.process.eUserName target_field: sentinel_one_cloud_funnel.event.src.process.e_user.name ignore_missing: true - set: + tag: set_process_user_name_20a920fd field: process.user.name copy_from: sentinel_one_cloud_funnel.event.src.process.e_user.name ignore_empty_value: true - rename: + tag: rename_json_tiIndicator_description_to_sentinel_one_cloud_funnel_event_ti_indicator_description_bef33971 field: json.tiIndicator.description target_field: sentinel_one_cloud_funnel.event.ti_indicator.description ignore_missing: true - set: + tag: set_threat_indicator_description_dc84c691 field: threat.indicator.description copy_from: sentinel_one_cloud_funnel.event.ti_indicator.description ignore_empty_value: true - rename: + tag: rename_json_tiIndicator_references_to_sentinel_one_cloud_funnel_event_ti_indicator_references_ae424c65 field: json.tiIndicator.references target_field: sentinel_one_cloud_funnel.event.ti_indicator.references ignore_missing: true - set: + tag: set_threat_indicator_reference_b10de2b8 field: threat.indicator.reference copy_from: sentinel_one_cloud_funnel.event.ti_indicator.references ignore_empty_value: true - rename: + tag: rename_json_tiIndicator_mitreTactics_to_sentinel_one_cloud_funnel_event_ti_indicator_mitre_tactics_203a2fd8 field: json.tiIndicator.mitreTactics target_field: sentinel_one_cloud_funnel.event.ti_indicator.mitre_tactics ignore_missing: true - append: + tag: append_threat_tactic_name_9a061c62 field: threat.tactic.name value: '{{{sentinel_one_cloud_funnel.event.ti_indicator.mitre_tactics}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.ti_indicator?.mitre_tactics != null - rename: + tag: rename_json_account_id_to_sentinel_one_cloud_funnel_event_account_id_e5884045 field: json.account.id target_field: sentinel_one_cloud_funnel.event.account_id ignore_missing: true - rename: + tag: rename_json_agent_uuid_to_sentinel_one_cloud_funnel_event_agent_uuid_d07ef584 field: json.agent.uuid target_field: sentinel_one_cloud_funnel.event.agent.uuid ignore_missing: true - set: + tag: set_host_id_c11022a6 field: host.id copy_from: sentinel_one_cloud_funnel.event.agent.uuid ignore_empty_value: true - rename: + tag: rename_json_agent_version_to_sentinel_one_cloud_funnel_event_agent_version_9eaa44e4 field: json.agent.version target_field: sentinel_one_cloud_funnel.event.agent.version ignore_missing: true - rename: + tag: rename_json_dataSource_name_to_sentinel_one_cloud_funnel_event_data_source_name_c309dd4f field: json.dataSource.name target_field: sentinel_one_cloud_funnel.event.data_source.name ignore_missing: true - rename: + tag: rename_json_event_category_to_sentinel_one_cloud_funnel_event_category_337784b2 field: json.event.category target_field: sentinel_one_cloud_funnel.event.category ignore_missing: true @@ -648,45 +771,56 @@ processors: if: ctx.json?.event?.repetitionCount != '' on_failure: - append: + tag: append_error_message_0cff8f50 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_event_type_to_sentinel_one_cloud_funnel_event_type_7099f006 field: json.event.type target_field: sentinel_one_cloud_funnel.event.type ignore_missing: true - rename: + tag: rename_json_i_scheme_to_sentinel_one_cloud_funnel_event_i_scheme_cfa7bb10 field: json.i.scheme target_field: sentinel_one_cloud_funnel.event.i.scheme ignore_missing: true - rename: + tag: rename_json_i_version_to_sentinel_one_cloud_funnel_event_i_version_26581e90 field: json.i.version target_field: sentinel_one_cloud_funnel.event.i.version ignore_missing: true - rename: + tag: rename_json_meta_event_name_to_sentinel_one_cloud_funnel_event_meta_event_name_c3ef4b96 field: json.meta.event.name target_field: sentinel_one_cloud_funnel.event.meta_event_name ignore_missing: true - rename: + tag: rename_json_mgmt_url_to_sentinel_one_cloud_funnel_event_mgmt_url_d92a68b4 field: json.mgmt.url target_field: sentinel_one_cloud_funnel.event.mgmt.url ignore_missing: true - rename: + tag: rename_json_osSrc_process_activeContent_hash_to_sentinel_one_cloud_funnel_event_os_src_process_active_content_hash_654c3063 field: json.osSrc.process.activeContent.hash target_field: sentinel_one_cloud_funnel.event.os_src_process.active_content.hash ignore_missing: true - rename: + tag: rename_json_osSrc_process_activeContent_id_to_sentinel_one_cloud_funnel_event_os_src_process_active_content_id_5892f61b field: json.osSrc.process.activeContent.id target_field: sentinel_one_cloud_funnel.event.os_src_process.active_content.id ignore_missing: true - rename: + tag: rename_json_osSrc_process_activeContent_path_to_sentinel_one_cloud_funnel_event_os_src_process_active_content_path_94f812b3 field: json.osSrc.process.activeContent.path target_field: sentinel_one_cloud_funnel.event.os_src_process.active_content.path ignore_missing: true - rename: + tag: rename_json_osSrc_process_activeContent_signedStatus_to_sentinel_one_cloud_funnel_event_os_src_process_active_content_signed_status_441f81f6 field: json.osSrc.process.activeContent.signedStatus target_field: sentinel_one_cloud_funnel.event.os_src_process.active_content.signed_status ignore_missing: true - rename: + tag: rename_json_osSrc_process_activeContentType_to_sentinel_one_cloud_funnel_event_os_src_process_active_content_type_ae46b6e3 field: json.osSrc.process.activeContentType target_field: sentinel_one_cloud_funnel.event.os_src_process.active_content.type ignore_missing: true @@ -699,9 +833,11 @@ processors: if: ctx.json?.osSrc?.process?.childProcCount != '' on_failure: - append: + tag: append_error_message_33130feb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_osSrc_process_cmdline_to_sentinel_one_cloud_funnel_event_os_src_process_cmd_line_e43262ab field: json.osSrc.process.cmdline target_field: sentinel_one_cloud_funnel.event.os_src_process.cmd_line ignore_missing: true @@ -714,6 +850,7 @@ processors: if: ctx.json?.osSrc?.process?.crossProcessCount != '' on_failure: - append: + tag: append_error_message_24900294 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -725,6 +862,7 @@ processors: if: ctx.json?.osSrc?.process?.crossProcessDupRemoteProcessHandleCount != '' on_failure: - append: + tag: append_error_message_1019465f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -736,6 +874,7 @@ processors: if: ctx.json?.osSrc?.process?.crossProcessDupThreadHandleCount != '' on_failure: - append: + tag: append_error_message_a5047704 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -747,6 +886,7 @@ processors: if: ctx.json?.osSrc?.process?.crossProcessOpenProcessCount != '' on_failure: - append: + tag: append_error_message_0dd3aed4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -758,6 +898,7 @@ processors: if: ctx.json?.osSrc?.process?.crossProcessOutOfStorylineCount != '' on_failure: - append: + tag: append_error_message_47dae6eb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -769,9 +910,11 @@ processors: if: ctx.json?.osSrc?.process?.crossProcessThreadCreateCount != '' on_failure: - append: + tag: append_error_message_fe3450ee field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_osSrc_process_displayName_to_sentinel_one_cloud_funnel_event_os_src_process_display_name_079bf379 field: json.osSrc.process.displayName target_field: sentinel_one_cloud_funnel.event.os_src_process.display_name ignore_missing: true @@ -784,6 +927,7 @@ processors: if: ctx.json?.osSrc?.process?.dnsCount != '' on_failure: - append: + tag: append_error_message_e1ac18f3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -795,35 +939,43 @@ processors: if: ctx.json?.osSrc?.process?.image?.binaryIsExecutable != '' on_failure: - append: + tag: append_error_message_fe408cf1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_osSrc_process_image_md5_to_sentinel_one_cloud_funnel_event_os_src_process_image_md5_345656d0 field: json.osSrc.process.image.md5 target_field: sentinel_one_cloud_funnel.event.os_src_process.image.md5 ignore_missing: true - append: + tag: append_related_hash_ef72b7b9 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.os_src_process.image.md5}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.os_src_process?.image?.md5!= null - rename: + tag: rename_json_osSrc_process_image_path_to_sentinel_one_cloud_funnel_event_os_src_process_image_path_4371ef3c field: json.osSrc.process.image.path target_field: sentinel_one_cloud_funnel.event.os_src_process.image.path ignore_missing: true - rename: + tag: rename_json_osSrc_process_image_sha1_to_sentinel_one_cloud_funnel_event_os_src_process_image_sha1_92d6200c field: json.osSrc.process.image.sha1 target_field: sentinel_one_cloud_funnel.event.os_src_process.image.sha1 ignore_missing: true - append: + tag: append_related_hash_ead1fad3 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.os_src_process.image.sha1}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.os_src_process?.image?.sha1 != null - rename: + tag: rename_json_osSrc_process_image_sha256_to_sentinel_one_cloud_funnel_event_os_src_process_image_sha256_f2c85f30 field: json.osSrc.process.image.sha256 target_field: sentinel_one_cloud_funnel.event.os_src_process.image.sha256 ignore_missing: true - append: + tag: append_related_hash_3e12afb7 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.os_src_process.image.sha256}}}' allow_duplicates: false @@ -837,6 +989,7 @@ processors: if: ctx.json?.osSrc?.process?.indicatorBootConfigurationUpdateCount != '' on_failure: - append: + tag: append_error_message_4a91a4b2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -848,6 +1001,7 @@ processors: if: ctx.json?.osSrc?.process?.indicatorEvasionCount != '' on_failure: - append: + tag: append_error_message_0636873a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -859,6 +1013,7 @@ processors: if: ctx.json?.osSrc?.process?.indicatorExploitationCount != '' on_failure: - append: + tag: append_error_message_ac8f33b0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -870,6 +1025,7 @@ processors: if: ctx.json?.osSrc?.process?.indicatorGeneral?.count != '' on_failure: - append: + tag: append_error_message_fdafaf1a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -881,6 +1037,7 @@ processors: if: ctx.json?.osSrc?.process?.indicatorInfostealerCount != '' on_failure: - append: + tag: append_error_message_ce870086 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -892,6 +1049,7 @@ processors: if: ctx.json?.osSrc?.process?.indicatorInjectionCount != '' on_failure: - append: + tag: append_error_message_6a4add8e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -903,6 +1061,7 @@ processors: if: ctx.json?.osSrc?.process?.indicatorPersistenceCount != '' on_failure: - append: + tag: append_error_message_1fe8aa26 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -914,6 +1073,7 @@ processors: if: ctx.json?.osSrc?.process?.indicatorPostExploitationCount != '' on_failure: - append: + tag: append_error_message_00b8aa2f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -925,6 +1085,7 @@ processors: if: ctx.json?.osSrc?.process?.indicatorRansomwareCount != '' on_failure: - append: + tag: append_error_message_18374ebc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -936,9 +1097,11 @@ processors: if: ctx.json?.osSrc?.process?.indicatorReconnaissanceCount != '' on_failure: - append: + tag: append_error_message_e58820bc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_osSrc_process_integrityLevel_to_sentinel_one_cloud_funnel_event_os_src_process_integrity_level_d358c9c7 field: json.osSrc.process.integrityLevel target_field: sentinel_one_cloud_funnel.event.os_src_process.integrity_level ignore_missing: true @@ -951,6 +1114,7 @@ processors: if: ctx.json?.osSrc?.process?.isNative64Bit != '' on_failure: - append: + tag: append_error_message_f6c481fe field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -962,6 +1126,7 @@ processors: if: ctx.json?.osSrc?.process?.isRedirectCmdProcessor != '' on_failure: - append: + tag: append_error_message_0ad45c90 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -973,6 +1138,7 @@ processors: if: ctx.json?.osSrc?.process?.isStorylineRoot != '' on_failure: - append: + tag: append_error_message_066ea83d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -984,9 +1150,11 @@ processors: if: ctx.json?.osSrc?.process?.moduleCount != '' on_failure: - append: + tag: append_error_message_03ab8d8e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_osSrc_process_name_to_sentinel_one_cloud_funnel_event_os_src_process_name_6be35882 field: json.osSrc.process.name target_field: sentinel_one_cloud_funnel.event.os_src_process.name ignore_missing: true @@ -999,6 +1167,7 @@ processors: if: ctx.json?.osSrc?.process?.netConnCount != '' on_failure: - append: + tag: append_error_message_299abb6a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1010,6 +1179,7 @@ processors: if: ctx.json?.osSrc?.process?.netConnInCount != '' on_failure: - append: + tag: append_error_message_41348bab field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1021,55 +1191,68 @@ processors: if: ctx.json?.osSrc?.process?.netConnOutCount != '' on_failure: - append: + tag: append_error_message_1e47b481 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_osSrc_process_parent_activeContent_hash_to_sentinel_one_cloud_funnel_event_os_src_process_parent_active_content_hash_519d4bad field: json.osSrc.process.parent.activeContent.hash target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.active_content.hash ignore_missing: true - rename: + tag: rename_json_osSrc_process_parent_activeContent_id_to_sentinel_one_cloud_funnel_event_os_src_process_parent_active_content_id_dca022ff field: json.osSrc.process.parent.activeContent.id target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.active_content.id ignore_missing: true - rename: + tag: rename_json_osSrc_process_parent_activeContent_path_to_sentinel_one_cloud_funnel_event_os_src_process_parent_active_content_path_61d5a0bf field: json.osSrc.process.parent.activeContent.path target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.active_content.path ignore_missing: true - rename: + tag: rename_json_osSrc_process_parent_activeContent_signedStatus_to_sentinel_one_cloud_funnel_event_os_src_process_parent_active_content_signed_status_df71e278 field: json.osSrc.process.parent.activeContent.signedStatus target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.active_content.signed_status ignore_missing: true - rename: + tag: rename_json_osSrc_process_parent_activeContentType_to_sentinel_one_cloud_funnel_event_os_src_process_parent_active_content_type_fb196e25 field: json.osSrc.process.parent.activeContentType target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.active_content.type ignore_missing: true - rename: + tag: rename_json_osSrc_process_parent_cmdline_to_sentinel_one_cloud_funnel_event_os_src_process_parent_cmd_line_1f7506cf field: json.osSrc.process.parent.cmdline target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.cmd_line ignore_missing: true - rename: + tag: rename_json_osSrc_process_parent_displayName_to_sentinel_one_cloud_funnel_event_os_src_process_parent_display_name_2a79afdb field: json.osSrc.process.parent.displayName target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.display_name ignore_missing: true - rename: + tag: rename_json_osSrc_process_parent_image_sha1_to_sentinel_one_cloud_funnel_event_os_src_process_parent_image_sha1_c7052b90 field: json.osSrc.process.parent.image.sha1 target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.image.sha1 ignore_missing: true - append: + tag: append_related_hash_50373d02 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.os_src_process.parent.image.sha1}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.os_src_process?.parent?.image?.sha1 != null - rename: + tag: rename_json_osSrc_process_parent_image_sha256_to_sentinel_one_cloud_funnel_event_os_src_process_parent_image_sha256_b14fbef4 field: json.osSrc.process.parent.image.sha256 target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.image.sha256 ignore_missing: true - append: + tag: append_related_hash_e29dcdce field: related.hash value: '{{{sentinel_one_cloud_funnel.event.os_src_process.parent.image.sha256}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.os_src_process?.parent?.image?.sha256 != null - rename: + tag: rename_json_osSrc_process_parent_integrityLevel_to_sentinel_one_cloud_funnel_event_os_src_process_parent_integrity_level_9f018fc1 field: json.osSrc.process.parent.integrityLevel target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.integrity_level ignore_missing: true @@ -1082,6 +1265,7 @@ processors: if: ctx.json?.osSrc?.process?.parent?.isNative64Bit != '' on_failure: - append: + tag: append_error_message_2d7d3e8a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1093,6 +1277,7 @@ processors: if: ctx.json?.osSrc?.process?.parent?.isRedirectCmdProcessor != '' on_failure: - append: + tag: append_error_message_c0e88dca field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1104,9 +1289,11 @@ processors: if: ctx.json?.osSrc?.process?.parent?.isStorylineRoot != '' on_failure: - append: + tag: append_error_message_a548f83b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_osSrc_process_parent_name_to_sentinel_one_cloud_funnel_event_os_src_process_parent_name_7e83fae4 field: json.osSrc.process.parent.name target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.name ignore_missing: true @@ -1119,13 +1306,16 @@ processors: if: ctx.json?.osSrc?.process?.parent?.pid != '' on_failure: - append: + tag: append_error_message_9bfc19b4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_osSrc_process_parent_publisher_to_sentinel_one_cloud_funnel_event_os_src_process_parent_publisher_e487e63c field: json.osSrc.process.parent.publisher target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.publisher ignore_missing: true - rename: + tag: rename_json_osSrc_process_parent_reasonSignatureInvalid_to_sentinel_one_cloud_funnel_event_os_src_process_parent_reason_signature_invalid_91dfccbc field: json.osSrc.process.parent.reasonSignatureInvalid target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.reason_signature_invalid ignore_missing: true @@ -1138,9 +1328,11 @@ processors: if: ctx.json?.osSrc?.process?.parent?.sessionId != '' on_failure: - append: + tag: append_error_message_4db5df8b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_osSrc_process_parent_signedStatus_to_sentinel_one_cloud_funnel_event_os_src_process_parent_signed_status_f052af77 field: json.osSrc.process.parent.signedStatus target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.signed_status ignore_missing: true @@ -1154,21 +1346,26 @@ processors: if: ctx.json?.osSrc?.process?.parent?.startTime != null && ctx.json.osSrc.process.parent.startTime != '' on_failure: - append: + tag: append_error_message_6606ce35 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_osSrc_process_parent_storyline_id_to_sentinel_one_cloud_funnel_event_os_src_process_parent_storyline_id_5abcd889 field: json.osSrc.process.parent.storyline.id target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.storyline_id ignore_missing: true - rename: + tag: rename_json_osSrc_process_parent_uid_to_sentinel_one_cloud_funnel_event_os_src_process_parent_uid_7d15cf0c field: json.osSrc.process.parent.uid target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.uid ignore_missing: true - rename: + tag: rename_json_osSrc_process_parent_user_to_sentinel_one_cloud_funnel_event_os_src_process_parent_user_name_a2786fb7 field: json.osSrc.process.parent.user target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.user.name ignore_missing: true - append: + tag: append_related_user_2934354d field: related.user value: '{{{sentinel_one_cloud_funnel.event.os_src_process.parent.user.name}}}' allow_duplicates: false @@ -1182,13 +1379,16 @@ processors: if: ctx.json?.osSrc?.process?.pid != '' on_failure: - append: + tag: append_error_message_65265912 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_osSrc_process_publisher_to_sentinel_one_cloud_funnel_event_os_src_process_publisher_96b223dc field: json.osSrc.process.publisher target_field: sentinel_one_cloud_funnel.event.os_src_process.publisher ignore_missing: true - rename: + tag: rename_json_osSrc_process_reasonSignatureInvalid_to_sentinel_one_cloud_funnel_event_os_src_process_reason_signature_invalid_37ea609a field: json.osSrc.process.reasonSignatureInvalid target_field: sentinel_one_cloud_funnel.event.os_src_process.reason_signature_invalid ignore_missing: true @@ -1201,6 +1401,7 @@ processors: if: ctx.json?.osSrc?.process?.registryChangeCount != '' on_failure: - append: + tag: append_error_message_ed6bbfef field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1212,9 +1413,11 @@ processors: if: ctx.json?.osSrc?.process?.sessionId != '' on_failure: - append: + tag: append_error_message_499af721 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_osSrc_process_signedStatus_to_sentinel_one_cloud_funnel_event_os_src_process_signed_status_2e4be41b field: json.osSrc.process.signedStatus target_field: sentinel_one_cloud_funnel.event.os_src_process.signed_status ignore_missing: true @@ -1228,13 +1431,16 @@ processors: if: ctx.json?.osSrc?.process?.startTime != null && ctx.json.osSrc.process.startTime != '' on_failure: - append: + tag: append_error_message_b9f33a8b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_osSrc_process_storyline_id_to_sentinel_one_cloud_funnel_event_os_src_process_storyline_id_f579c25d field: json.osSrc.process.storyline.id target_field: sentinel_one_cloud_funnel.event.os_src_process.storyline_id ignore_missing: true - rename: + tag: rename_json_osSrc_process_subsystem_to_sentinel_one_cloud_funnel_event_os_src_process_subsystem_04548168 field: json.osSrc.process.subsystem target_field: sentinel_one_cloud_funnel.event.os_src_process.subsystem ignore_missing: true @@ -1247,6 +1453,7 @@ processors: if: ctx.json?.osSrc?.process?.tgtFileCreationCount != '' on_failure: - append: + tag: append_error_message_3705fcdb field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1258,6 +1465,7 @@ processors: if: ctx.json?.osSrc?.process?.tgtFileDeletionCount != '' on_failure: - append: + tag: append_error_message_adcc78d9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1269,22 +1477,27 @@ processors: if: ctx.json?.osSrc?.process?.tgtFileModificationCount != '' on_failure: - append: + tag: append_error_message_2deb8d6d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_osSrc_process_uid_to_sentinel_one_cloud_funnel_event_os_src_process_uid_1ba8c838 field: json.osSrc.process.uid target_field: sentinel_one_cloud_funnel.event.os_src_process.uid ignore_missing: true - rename: + tag: rename_json_osSrc_process_user_to_sentinel_one_cloud_funnel_event_os_src_process_user_name_0406abf9 field: json.osSrc.process.user target_field: sentinel_one_cloud_funnel.event.os_src_process.user.name ignore_missing: true - append: + tag: append_related_user_6f913a82 field: related.user value: '{{{sentinel_one_cloud_funnel.event.os_src_process.user.name}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.os_src_process?.user?.name != null - rename: + tag: rename_json_osSrc_process_verifiedStatus_to_sentinel_one_cloud_funnel_event_os_src_process_verified_status_4b19d273 field: json.osSrc.process.verifiedStatus target_field: sentinel_one_cloud_funnel.event.os_src_process.verified_status ignore_missing: true @@ -1298,33 +1511,41 @@ processors: if: ctx.json != null && ctx.json['sca:atlantisIngestTime'] != null && ctx.json['sca:atlantisIngestTime'] != '' on_failure: - append: + tag: append_error_message_0a33048c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_site_id_to_sentinel_one_cloud_funnel_event_site_id_45a0ad1e field: json.site.id target_field: sentinel_one_cloud_funnel.event.site.id ignore_missing: true - rename: + tag: rename_json_site_name_to_sentinel_one_cloud_funnel_event_site_name_b98f15e6 field: json.site.name target_field: sentinel_one_cloud_funnel.event.site.name ignore_missing: true - rename: + tag: rename_json_src_process_activeContent_hash_to_sentinel_one_cloud_funnel_event_src_process_active_content_hash_1fb5acb1 field: json.src.process.activeContent.hash target_field: sentinel_one_cloud_funnel.event.src.process.active_content.hash ignore_missing: true - rename: + tag: rename_json_src_process_activeContent_id_to_sentinel_one_cloud_funnel_event_src_process_active_content_id_c22f443b field: json.src.process.activeContent.id target_field: sentinel_one_cloud_funnel.event.src.process.active_content.id ignore_missing: true - rename: + tag: rename_json_src_process_activeContent_path_to_sentinel_one_cloud_funnel_event_src_process_active_content_path_7bca1543 field: json.src.process.activeContent.path target_field: sentinel_one_cloud_funnel.event.src.process.active_content.path ignore_missing: true - rename: + tag: rename_json_src_process_activeContent_signedStatus_to_sentinel_one_cloud_funnel_event_src_process_active_content_signed_status_bdf13cd4 field: json.src.process.activeContent.signedStatus target_field: sentinel_one_cloud_funnel.event.src.process.active_content.signed_status ignore_missing: true - rename: + tag: rename_json_src_process_activeContentType_to_sentinel_one_cloud_funnel_event_src_process_active_content_type_d627bae9 field: json.src.process.activeContentType target_field: sentinel_one_cloud_funnel.event.src.process.active_content.type ignore_missing: true @@ -1337,6 +1558,7 @@ processors: if: ctx.json?.src?.process?.childProcCount != '' on_failure: - append: + tag: append_error_message_2b5fc00f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1348,6 +1570,7 @@ processors: if: ctx.json?.src?.process?.crossProcessDupRemoteProcessHandleCount != '' on_failure: - append: + tag: append_error_message_15c1a8cd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1359,6 +1582,7 @@ processors: if: ctx.json?.src?.process?.crossProcessOpenProcessCount != '' on_failure: - append: + tag: append_error_message_3d119178 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1370,6 +1594,7 @@ processors: if: ctx.json?.src?.process?.indicatorExploitationCount != '' on_failure: - append: + tag: append_error_message_53877b26 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1381,6 +1606,7 @@ processors: if: ctx.json?.src?.process?.indicatorInjectionCount != '' on_failure: - append: + tag: append_error_message_0bbe1012 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1392,6 +1618,7 @@ processors: if: ctx.json?.src?.process?.indicatorRansomwareCount != '' on_failure: - append: + tag: append_error_message_dae80fe0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1403,17 +1630,21 @@ processors: if: ctx.json?.src?.process?.indicatorReconnaissanceCount != '' on_failure: - append: + tag: append_error_message_a17af302 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_src_process_integrityLevel_to_sentinel_one_cloud_funnel_event_src_process_integrity_level_942358f5 field: json.src.process.integrityLevel target_field: sentinel_one_cloud_funnel.event.src.process.integrity_level ignore_missing: true - rename: + tag: rename_json_src_process_lUserName_to_sentinel_one_cloud_funnel_event_src_process_l_user_name_b2c094b5 field: json.src.process.lUserName target_field: sentinel_one_cloud_funnel.event.src.process.l_user.name ignore_missing: true - append: + tag: append_related_user_6b894131 field: related.user value: '{{{sentinel_one_cloud_funnel.event.src.process.l_user.name}}}' allow_duplicates: false @@ -1427,9 +1658,11 @@ processors: if: ctx.json?.src?.process?.lUserUid != '' on_failure: - append: + tag: append_error_message_c1c67e8b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: + tag: append_related_user_c782b557 field: related.user value: '{{{sentinel_one_cloud_funnel.event.src.process.l_user.uid}}}' allow_duplicates: false @@ -1443,6 +1676,7 @@ processors: if: ctx.json?.src?.process?.moduleCount != '' on_failure: - append: + tag: append_error_message_25fb87ca field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1454,46 +1688,57 @@ processors: if: ctx.json?.src?.process?.netConnInCount != '' on_failure: - append: + tag: append_error_message_fe4efb8f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_src_process_parent_activeContent_hash_to_sentinel_one_cloud_funnel_event_src_process_parent_active_content_hash_4fc0076b field: json.src.process.parent.activeContent.hash target_field: sentinel_one_cloud_funnel.event.src.process.parent.active_content.hash ignore_missing: true - rename: + tag: rename_json_src_process_parent_activeContent_id_to_sentinel_one_cloud_funnel_event_src_process_parent_active_content_id_01479e37 field: json.src.process.parent.activeContent.id target_field: sentinel_one_cloud_funnel.event.src.process.parent.active_content.id ignore_missing: true - rename: + tag: rename_json_src_process_parent_activeContent_path_to_sentinel_one_cloud_funnel_event_src_process_parent_active_content_path_92fca66f field: json.src.process.parent.activeContent.path target_field: sentinel_one_cloud_funnel.event.src.process.parent.active_content.path ignore_missing: true - rename: + tag: rename_json_src_process_parent_activeContent_signedStatus_to_sentinel_one_cloud_funnel_event_src_process_parent_active_content_signed_status_8d329f26 field: json.src.process.parent.activeContent.signedStatus target_field: sentinel_one_cloud_funnel.event.src.process.parent.active_content.signed_status ignore_missing: true - rename: + tag: rename_json_src_process_parent_activeContentType_to_sentinel_one_cloud_funnel_event_src_process_parent_active_content_type_7aadf53b field: json.src.process.parent.activeContentType target_field: sentinel_one_cloud_funnel.event.src.process.parent.active_content.type ignore_missing: true - rename: + tag: rename_json_src_process_parent_image_md5_to_sentinel_one_cloud_funnel_event_src_process_parent_image_md5_9cfe85f8 field: json.src.process.parent.image.md5 target_field: sentinel_one_cloud_funnel.event.src.process.parent.image.md5 ignore_missing: true - append: + tag: append_related_hash_aa10452f field: related.hash value: '{{{sentinel_one_cloud_funnel.event.src.process.parent.image.md5}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.src?.process?.parent?.image?.md5 != null - rename: + tag: rename_json_src_process_parent_image_path_to_sentinel_one_cloud_funnel_event_src_process_parent_image_path_d038e848 field: json.src.process.parent.image.path target_field: sentinel_one_cloud_funnel.event.src.process.parent.image.path ignore_missing: true - set: + tag: set_process_parent_executable_ca714543 field: process.parent.executable copy_from: sentinel_one_cloud_funnel.event.src.process.parent.image.path ignore_empty_value: true - rename: + tag: rename_json_src_process_parent_integrityLevel_to_sentinel_one_cloud_funnel_event_src_process_parent_integrity_level_d96f7bbf field: json.src.process.parent.integrityLevel target_field: sentinel_one_cloud_funnel.event.src.process.parent.integrity_level ignore_missing: true @@ -1506,6 +1751,7 @@ processors: if: ctx.json?.src?.process?.parent?.isNative64Bit != '' on_failure: - append: + tag: append_error_message_b687e9b0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1517,6 +1763,7 @@ processors: if: ctx.json?.src?.process?.parent?.isRedirectCmdProcessor != '' on_failure: - append: + tag: append_error_message_88d91c20 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1528,13 +1775,16 @@ processors: if: ctx.json?.src?.process?.parent?.isStorylineRoot != '' on_failure: - append: + tag: append_error_message_154afb8f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_src_process_parent_lUserName_to_sentinel_one_cloud_funnel_event_src_process_parent_l_user_name_b9ea45d5 field: json.src.process.parent.lUserName target_field: sentinel_one_cloud_funnel.event.src.process.parent.l_user.name ignore_missing: true - append: + tag: append_related_user_420cce14 field: related.user value: '{{{sentinel_one_cloud_funnel.event.src.process.parent.l_user.name}}}' allow_duplicates: false @@ -1548,18 +1798,22 @@ processors: if: ctx.json?.src?.process?.parent?.lUserUid != '' on_failure: - append: + tag: append_error_message_bbfcbd73 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: + tag: append_related_user_15a4a06c field: related.user value: '{{{sentinel_one_cloud_funnel.event.src.process.parent.l_user.uid}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.src?.process?.parent?.l_user?.uid != null - rename: + tag: rename_json_src_process_parent_publisher_to_sentinel_one_cloud_funnel_event_src_process_parent_publisher_817e96a0 field: json.src.process.parent.publisher target_field: sentinel_one_cloud_funnel.event.src.process.parent.publisher ignore_missing: true - rename: + tag: rename_json_src_process_parent_reasonSignatureInvalid_to_sentinel_one_cloud_funnel_event_src_process_parent_reason_signature_invalid_3a0c5d6a field: json.src.process.parent.reasonSignatureInvalid target_field: sentinel_one_cloud_funnel.event.src.process.parent.reason_signature_invalid ignore_missing: true @@ -1572,38 +1826,47 @@ processors: if: ctx.json?.src?.process?.parent?.sessionId != '' on_failure: - append: + tag: append_error_message_f7133fa7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_src_process_parent_signedStatus_to_sentinel_one_cloud_funnel_event_src_process_parent_signed_status_05be8757 field: json.src.process.parent.signedStatus target_field: sentinel_one_cloud_funnel.event.src.process.parent.signed_status ignore_missing: true - rename: + tag: rename_json_src_process_parent_storyline_id_to_sentinel_one_cloud_funnel_event_src_process_parent_storyline_id_7f089789 field: json.src.process.parent.storyline.id target_field: sentinel_one_cloud_funnel.event.src.process.parent.storyline_id ignore_missing: true - rename: + tag: rename_json_src_process_parent_uid_to_sentinel_one_cloud_funnel_event_src_process_parent_uid_606a28e4 field: json.src.process.parent.uid target_field: sentinel_one_cloud_funnel.event.src.process.parent.uid ignore_missing: true - set: + tag: set_process_parent_entity_id_61c5c0de field: process.parent.entity_id copy_from: sentinel_one_cloud_funnel.event.src.process.parent.uid ignore_empty_value: true - rename: + tag: rename_json_src_process_parent_userSid_to_sentinel_one_cloud_funnel_event_src_process_parent_user_sid_0443a94a field: json.src.process.parent.userSid target_field: sentinel_one_cloud_funnel.event.src.process.parent.user.sid ignore_missing: true - append: + tag: append_related_user_0ddaaeec field: related.user value: '{{{sentinel_one_cloud_funnel.event.src.process.parent.user.sid}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.src?.process?.parent?.user?.sid != null - rename: + tag: rename_json_src_process_storyline_id_to_sentinel_one_cloud_funnel_event_src_process_storyline_id_81831145 field: json.src.process.storyline.id target_field: sentinel_one_cloud_funnel.event.src.process.storyline_id ignore_missing: true - rename: + tag: rename_json_src_process_subsystem_to_sentinel_one_cloud_funnel_event_src_process_subsystem_b1655bce field: json.src.process.subsystem target_field: sentinel_one_cloud_funnel.event.src.process.subsystem ignore_missing: true @@ -1616,17 +1879,21 @@ processors: if: ctx.json?.src?.process?.tgtFileCreationCount != '' on_failure: - append: + tag: append_error_message_295c0ca7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tgt_file_isSigned_to_sentinel_one_cloud_funnel_event_tgt_file_is_signed_093a7793 field: json.tgt.file.isSigned target_field: sentinel_one_cloud_funnel.event.tgt.file.is_signed ignore_missing: true - rename: + tag: rename_json_tgt_process_eUserName_to_sentinel_one_cloud_funnel_event_tgt_process_e_user_name_f5d59305 field: json.tgt.process.eUserName target_field: sentinel_one_cloud_funnel.event.tgt.process.e_user.name ignore_missing: true - append: + tag: append_related_user_f64876e5 field: related.user value: '{{{sentinel_one_cloud_funnel.event.tgt.process.e_user.name}}}' allow_duplicates: false @@ -1640,18 +1907,22 @@ processors: if: ctx.json?.tgt?.process?.eUserUid != '' on_failure: - append: + tag: append_error_message_5652219b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: + tag: append_related_user_3bee7c6f field: related.user value: '{{{sentinel_one_cloud_funnel.event.tgt.process.e_user.uid}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.process?.e_user?.uid != null - rename: + tag: rename_json_tgt_process_lUserName_to_sentinel_one_cloud_funnel_event_tgt_process_l_user_name_00afd15f field: json.tgt.process.lUserName target_field: sentinel_one_cloud_funnel.event.tgt.process.l_user.name ignore_missing: true - append: + tag: append_related_user_3d7736a5 field: related.user value: '{{{sentinel_one_cloud_funnel.event.tgt.process.l_user.name}}}' allow_duplicates: false @@ -1665,18 +1936,22 @@ processors: if: ctx.json?.tgt?.process?.lUserUid != '' on_failure: - append: + tag: append_error_message_5016ef83 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: + tag: append_related_user_6c3e5bcd field: related.user value: '{{{sentinel_one_cloud_funnel.event.tgt.process.l_user.uid}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.process?.l_user?.uid != null - rename: + tag: rename_json_tgt_process_rUserName_to_sentinel_one_cloud_funnel_event_tgt_process_r_user_name_901383cf field: json.tgt.process.rUserName target_field: sentinel_one_cloud_funnel.event.tgt.process.r_user.name ignore_missing: true - append: + tag: append_related_user_472b3c8d field: related.user value: '{{{sentinel_one_cloud_funnel.event.tgt.process.r_user.name}}}' allow_duplicates: false @@ -1690,9 +1965,11 @@ processors: if: ctx.json?.tgt?.process?.rUserUid != '' on_failure: - append: + tag: append_error_message_aad8ba87 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: + tag: append_related_user_741d6399 field: related.user value: '{{{sentinel_one_cloud_funnel.event.tgt.process.r_user.uid}}}' allow_duplicates: false @@ -1706,9 +1983,11 @@ processors: if: ctx.json?.src?.process?.tid != '' on_failure: - append: + tag: append_error_message_114b3353 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_process_thread_id_95a0158f field: process.thread.id copy_from: sentinel_one_cloud_funnel.event.src.process.tid ignore_empty_value: true @@ -1723,21 +2002,26 @@ processors: if: ctx.json?.timestamp != null && ctx.json.timestamp != '' on_failure: - append: + tag: append_error_message_ab89ef75 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_dataSource_category_to_sentinel_one_cloud_funnel_event_data_source_category_c32e9aff field: json.dataSource.category target_field: sentinel_one_cloud_funnel.event.data_source.category ignore_missing: true - rename: + tag: rename_json_mgmt_id_to_sentinel_one_cloud_funnel_event_mgmt_id_d639682a field: json.mgmt.id target_field: sentinel_one_cloud_funnel.event.mgmt.id ignore_missing: true - rename: + tag: rename_json_mgmt_osRevision_to_sentinel_one_cloud_funnel_event_mgmt_os_revision_55a7a193 field: json.mgmt.osRevision target_field: sentinel_one_cloud_funnel.event.mgmt.os_revision ignore_missing: true - rename: + tag: rename_json_process_unique_key_to_sentinel_one_cloud_funnel_event_process_unique_key_f9eeee32 field: json.process.unique.key target_field: sentinel_one_cloud_funnel.event.process_unique_key ignore_missing: true @@ -1751,6 +2035,7 @@ processors: if: ctx.json != null && ctx.json['sca:ingestTime'] != null && ctx.json['sca:ingestTime'] != '' on_failure: - append: + tag: append_error_message_e6281510 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1762,6 +2047,7 @@ processors: if: ctx.json?.src?.process?.crossProcessDupThreadHandleCount != '' on_failure: - append: + tag: append_error_message_2b90af30 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1773,6 +2059,7 @@ processors: if: ctx.json?.src?.process?.dnsCount != '' on_failure: - append: + tag: append_error_message_dabf3440 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1784,6 +2071,7 @@ processors: if: ctx.json?.src?.process?.indicatorBootConfigurationUpdateCount != '' on_failure: - append: + tag: append_error_message_10b0dc56 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1795,6 +2083,7 @@ processors: if: ctx.json?.src?.process?.indicatorEvasionCount != '' on_failure: - append: + tag: append_error_message_1f963a9e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1806,6 +2095,7 @@ processors: if: ctx.json?.src?.process?.indicatorGeneralCount != '' on_failure: - append: + tag: append_error_message_a685513c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1817,6 +2107,7 @@ processors: if: ctx.json?.src?.process?.indicatorInfostealerCount != '' on_failure: - append: + tag: append_error_message_e85af115 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1828,6 +2119,7 @@ processors: if: ctx.json?.src?.process?.indicatorPersistenceCount != '' on_failure: - append: + tag: append_error_message_e8e7d17a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1839,6 +2131,7 @@ processors: if: ctx.json?.src?.process?.isNative64Bit != '' on_failure: - append: + tag: append_error_message_02f8842c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1850,6 +2143,7 @@ processors: if: ctx.json?.src?.process?.isRedirectCmdProcessor != '' on_failure: - append: + tag: append_error_message_69e5868e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1861,6 +2155,7 @@ processors: if: ctx.json?.src?.process?.netConnCount != '' on_failure: - append: + tag: append_error_message_0d231d08 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1872,9 +2167,11 @@ processors: if: ctx.json?.src?.process?.netConnOutCount != '' on_failure: - append: + tag: append_error_message_aa0dc0bf field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_src_process_parent_subsystem_to_sentinel_one_cloud_funnel_event_src_process_parent_subsystem_54cb1e10 field: json.src.process.parent.subsystem target_field: sentinel_one_cloud_funnel.event.src.process.parent.subsystem ignore_missing: true @@ -1887,6 +2184,7 @@ processors: if: ctx.json?.src?.process?.registryChangeCount != '' on_failure: - append: + tag: append_error_message_da1f3b29 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1898,17 +2196,21 @@ processors: if: ctx.json?.src?.process?.sessionId != '' on_failure: - append: + tag: append_error_message_8723148d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_src_process_signedStatus_to_sentinel_one_cloud_funnel_event_src_process_signed_status_c3b42fa3 field: json.src.process.signedStatus target_field: sentinel_one_cloud_funnel.event.src.process.signed_status ignore_missing: true - set: + tag: set_process_code_signature_exists_22576671 field: process.code_signature.exists value: true if: ctx.sentinel_one_cloud_funnel?.event?.src?.process?.signed_status == 'signed' - set: + tag: set_process_code_signature_exists_35a12473 field: process.code_signature.exists value: false if: ctx.sentinel_one_cloud_funnel?.event?.src?.process?.signed_status == 'unsigned' @@ -1921,6 +2223,7 @@ processors: if: ctx.json?.src?.process?.tgtFileDeletionCount != '' on_failure: - append: + tag: append_error_message_e114773b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1932,21 +2235,26 @@ processors: if: ctx.json?.src?.process?.tgtFileModificationCount != '' on_failure: - append: + tag: append_error_message_4fc02d6b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_src_process_uid_to_sentinel_one_cloud_funnel_event_src_process_uid_bc3170b8 field: json.src.process.uid target_field: sentinel_one_cloud_funnel.event.src.process.uid ignore_missing: true - set: + tag: set_process_entity_id_1c307620 field: process.entity_id copy_from: sentinel_one_cloud_funnel.event.src.process.uid ignore_empty_value: true - rename: + tag: rename_json_packet_id_to_sentinel_one_cloud_funnel_event_packet_id_06ce1e05 field: json.packet.id target_field: sentinel_one_cloud_funnel.event.packet_id ignore_missing: true - rename: + tag: rename_json_trace_id_to_sentinel_one_cloud_funnel_event_trace_id_a786e2c9 field: json.trace.id target_field: sentinel_one_cloud_funnel.event.trace_id ignore_missing: true @@ -1959,6 +2267,7 @@ processors: if: ctx.json?.src?.process?.crossProcessCount != '' on_failure: - append: + tag: append_error_message_6e501c82 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1970,6 +2279,7 @@ processors: if: ctx.json?.src?.process?.crossProcessOutOfStorylineCount != '' on_failure: - append: + tag: append_error_message_359e14ad field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1981,6 +2291,7 @@ processors: if: ctx.json?.src?.process?.crossProcessThreadCreateCount != '' on_failure: - append: + tag: append_error_message_68af7fb8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -1992,6 +2303,7 @@ processors: if: ctx.json?.src?.process?.image?.binaryIsExecutable != '' on_failure: - append: + tag: append_error_message_daf0310f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2003,6 +2315,7 @@ processors: if: ctx.json?.src?.process?.indicatorPostExploitationCount != '' on_failure: - append: + tag: append_error_message_fe805505 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2014,30 +2327,37 @@ processors: if: ctx.json?.src?.process?.isStorylineRoot != '' on_failure: - append: + tag: append_error_message_8c922bb9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_src_process_publisher_to_sentinel_one_cloud_funnel_event_src_process_publisher_d84f6a90 field: json.src.process.publisher target_field: sentinel_one_cloud_funnel.event.src.process.publisher ignore_missing: true - set: + tag: set_process_code_signature_subject_name_43fc49bd field: process.code_signature.subject_name copy_from: sentinel_one_cloud_funnel.event.src.process.publisher ignore_empty_value: true - rename: + tag: rename_json_src_process_verifiedStatus_to_sentinel_one_cloud_funnel_event_src_process_verified_status_be70825b field: json.src.process.verifiedStatus target_field: sentinel_one_cloud_funnel.event.src.process.verified_status ignore_missing: true - set: + tag: set_process_code_signature_trusted_8834435b field: process.code_signature.trusted value: true if: ctx.sentinel_one_cloud_funnel?.event?.src?.process?.verified_status?.contains('verified') == true - set: + tag: set_process_code_signature_trusted_88027e8b field: process.code_signature.trusted value: false override: false if: ctx.process?.code_signature?.exists == true - rename: + tag: rename_json_driver_certificate_thumbprint_to_sentinel_one_cloud_funnel_event_driver_certificate_thumbprint_value_65098dd7 field: json.driver.certificate.thumbprint target_field: sentinel_one_cloud_funnel.event.driver.certificate.thumbprint.value ignore_missing: true @@ -2050,6 +2370,7 @@ processors: if: ctx.json?.driver?.certificate?.thumbprintAlgorithm != '' on_failure: - append: + tag: append_error_message_262c8b33 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2061,57 +2382,70 @@ processors: if: ctx.json?.driver?.isLoadedBeforeMonitor != '' on_failure: - append: + tag: append_error_message_a5d4759a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_driver_loadVerdict_to_sentinel_one_cloud_funnel_event_driver_load_verdict_4e18b567 field: json.driver.loadVerdict target_field: sentinel_one_cloud_funnel.event.driver.load_verdict ignore_missing: true - rename: + tag: rename_json_driver_peSha1_to_sentinel_one_cloud_funnel_event_driver_pe_sha1_b7d01d1a field: json.driver.peSha1 target_field: sentinel_one_cloud_funnel.event.driver.pe.sha1 ignore_missing: true - append: + tag: append_related_hash_b8deff85 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.driver.pe.sha1}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.driver?.pe?.sha1 != null - rename: + tag: rename_json_driver_peSha256_to_sentinel_one_cloud_funnel_event_driver_pe_sha256_a31194ce field: json.driver.peSha256 target_field: sentinel_one_cloud_funnel.event.driver.pe.sha256 ignore_missing: true - append: + tag: append_related_hash_abdd3959 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.driver.pe.sha256}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.driver?.pe?.sha256 != null - rename: + tag: rename_json_driver_startType_to_sentinel_one_cloud_funnel_event_driver_start_type_9a8e8703 field: json.driver.startType target_field: sentinel_one_cloud_funnel.event.driver.start_type ignore_missing: true - rename: + tag: rename_json_event_dns_status_to_sentinel_one_cloud_funnel_event_dns_status_aa21af52 field: json.event.dns.status target_field: sentinel_one_cloud_funnel.event.dns.status ignore_missing: true - rename: + tag: rename_json_event_login_tgt_userSid_to_sentinel_one_cloud_funnel_event_login_tgt_user_sid_9cef4990 field: json.event.login.tgt.userSid target_field: sentinel_one_cloud_funnel.event.login.tgt.user.sid ignore_missing: true - append: + tag: append_related_user_e5b8f6cf field: related.user value: '{{{sentinel_one_cloud_funnel.event.login.tgt.user.sid}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.login?.tgt?.user?.sid != null - rename: + tag: rename_json_event_logout_tgt_userSid_to_sentinel_one_cloud_funnel_event_logout_tgt_user_sid_02b01034 field: json.event.logout.tgt.userSid target_field: sentinel_one_cloud_funnel.event.logout.tgt.user.sid ignore_missing: true - append: + tag: append_related_user_e5748945 field: related.user value: '{{{sentinel_one_cloud_funnel.event.logout.tgt.user.sid}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.logout?.tgt?.user?.sid != null - rename: + tag: rename_json_event_logout_type_to_sentinel_one_cloud_funnel_event_logout_type_244b24f4 field: json.event.logout.type target_field: sentinel_one_cloud_funnel.event.logout.type ignore_missing: true @@ -2124,29 +2458,36 @@ processors: if: ctx.json?.event?.processtermination?.exitCode != '' on_failure: - append: + tag: append_error_message_c71eb01c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_event_processtermination_signal_to_sentinel_one_cloud_funnel_event_process_termination_signal_fcdae261 field: json.event.processtermination.signal target_field: sentinel_one_cloud_funnel.event.process_termination.signal ignore_missing: true - rename: + tag: rename_json_event_url_source_to_sentinel_one_cloud_funnel_event_url_source_e91f6cc6 field: json.event.url.source target_field: sentinel_one_cloud_funnel.event.url.source ignore_missing: true - rename: + tag: rename_json_group_type_to_sentinel_one_cloud_funnel_event_group_type_e2040140 field: json.group.type target_field: sentinel_one_cloud_funnel.event.group.type ignore_missing: true - rename: + tag: rename_json_indicator_identifier_to_sentinel_one_cloud_funnel_event_indicator_identifier_82cfff90 field: json.indicator.identifier target_field: sentinel_one_cloud_funnel.event.indicator.identifier ignore_missing: true - rename: + tag: rename_json_namedPipe_accessMode_to_sentinel_one_cloud_funnel_event_named_pipe_access_mode_bbdc0958 field: json.namedPipe.accessMode target_field: sentinel_one_cloud_funnel.event.named_pipe.access_mode ignore_missing: true - rename: + tag: rename_json_namedPipe_connectionType_to_sentinel_one_cloud_funnel_event_named_pipe_connection_type_32be061c field: json.namedPipe.connectionType target_field: sentinel_one_cloud_funnel.event.named_pipe.connection_type ignore_missing: true @@ -2159,6 +2500,7 @@ processors: if: ctx.json?.namedPipe?.isFirstInstance != '' on_failure: - append: + tag: append_error_message_0ed51ecc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2170,6 +2512,7 @@ processors: if: ctx.json?.namedPipe?.isOverlapped != '' on_failure: - append: + tag: append_error_message_6dbd80c7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2181,6 +2524,7 @@ processors: if: ctx.json?.namedPipe?.isWriteThrough != '' on_failure: - append: + tag: append_error_message_8693e07c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2192,41 +2536,51 @@ processors: if: ctx.json?.namedPipe?.maxInstances != '' on_failure: - append: + tag: append_error_message_8307e425 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_namedPipe_name_to_sentinel_one_cloud_funnel_event_named_pipe_name_b5d50f3b field: json.namedPipe.name target_field: sentinel_one_cloud_funnel.event.named_pipe.name ignore_missing: true - rename: + tag: rename_json_namedPipe_readMode_to_sentinel_one_cloud_funnel_event_named_pipe_read_mode_48bea7a4 field: json.namedPipe.readMode target_field: sentinel_one_cloud_funnel.event.named_pipe.read_mode ignore_missing: true - rename: + tag: rename_json_namedPipe_remoteClients_to_sentinel_one_cloud_funnel_event_named_pipe_remote_clients_ddcb4268 field: json.namedPipe.remoteClients target_field: sentinel_one_cloud_funnel.event.named_pipe.remote_clients ignore_missing: true - rename: + tag: rename_json_namedPipe_securityGroups_to_sentinel_one_cloud_funnel_event_named_pipe_security_groups_e002bd65 field: json.namedPipe.securityGroups target_field: sentinel_one_cloud_funnel.event.named_pipe.security.groups ignore_missing: true - rename: + tag: rename_json_namedPipe_securityOwner_to_sentinel_one_cloud_funnel_event_named_pipe_security_owner_2ef45689 field: json.namedPipe.securityOwner target_field: sentinel_one_cloud_funnel.event.named_pipe.security.owner ignore_missing: true - rename: + tag: rename_json_namedPipe_typeMode_to_sentinel_one_cloud_funnel_event_named_pipe_type_mode_166759f4 field: json.namedPipe.typeMode target_field: sentinel_one_cloud_funnel.event.named_pipe.type_mode ignore_missing: true - rename: + tag: rename_json_namedPipe_waitMode_to_sentinel_one_cloud_funnel_event_named_pipe_wait_mode_5ba6365e field: json.namedPipe.waitMode target_field: sentinel_one_cloud_funnel.event.named_pipe.wait_mode ignore_missing: true - rename: + tag: rename_json_osSrc_process_image_extension_to_sentinel_one_cloud_funnel_event_os_src_process_image_extension_e77bec80 field: json.osSrc.process.image.extension target_field: sentinel_one_cloud_funnel.event.os_src_process.image.extension ignore_missing: true - rename: + tag: rename_json_osSrc_process_image_location_to_sentinel_one_cloud_funnel_event_os_src_process_image_location_dda518ec field: json.osSrc.process.image.location target_field: sentinel_one_cloud_funnel.event.os_src_process.image.location ignore_missing: true @@ -2239,6 +2593,7 @@ processors: if: ctx.json?.osSrc?.process?.image?.signature?.isValid != '' on_failure: - append: + tag: append_error_message_19ee9553 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2250,13 +2605,16 @@ processors: if: ctx.json?.osSrc?.process?.image?.size != '' on_failure: - append: + tag: append_error_message_06a989a3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_osSrc_process_image_type_to_sentinel_one_cloud_funnel_event_os_src_process_image_type_966170ca field: json.osSrc.process.image.type target_field: sentinel_one_cloud_funnel.event.os_src_process.image.type ignore_missing: true - rename: + tag: rename_json_osSrc_process_image_uid_to_sentinel_one_cloud_funnel_event_os_src_process_image_uid_bb9dbe38 field: json.osSrc.process.image.uid target_field: sentinel_one_cloud_funnel.event.os_src_process.image.uid ignore_missing: true @@ -2269,13 +2627,16 @@ processors: if: ctx.json?.osSrc?.process?.parent?.image?.binaryIsExecutable != '' on_failure: - append: + tag: append_error_message_f3e4f443 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_osSrc_process_parent_image_extension_to_sentinel_one_cloud_funnel_event_os_src_process_parent_image_extension_3333db14 field: json.osSrc.process.parent.image.extension target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.image.extension ignore_missing: true - rename: + tag: rename_json_osSrc_process_parent_image_location_to_sentinel_one_cloud_funnel_event_os_src_process_parent_image_location_d453aab4 field: json.osSrc.process.parent.image.location target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.image.location ignore_missing: true @@ -2288,6 +2649,7 @@ processors: if: ctx.json?.osSrc?.process?.parent?.image?.signature?.isValid != '' on_failure: - append: + tag: append_error_message_0b480ed5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2299,60 +2661,74 @@ processors: if: ctx.json?.osSrc?.process?.parent?.image?.size != '' on_failure: - append: + tag: append_error_message_077f8251 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_osSrc_process_parent_image_type_to_sentinel_one_cloud_funnel_event_os_src_process_parent_image_type_2dcf4dfc field: json.osSrc.process.parent.image.type target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.image.type ignore_missing: true - rename: + tag: rename_json_osSrc_process_parent_image_uid_to_sentinel_one_cloud_funnel_event_os_src_process_parent_image_uid_a453fe12 field: json.osSrc.process.parent.image.uid target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.image.uid ignore_missing: true - rename: + tag: rename_json_osSrc_process_parent_subsystem_to_sentinel_one_cloud_funnel_event_os_src_process_parent_subsystem_990462da field: json.osSrc.process.parent.subsystem target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.subsystem ignore_missing: true - rename: + tag: rename_json_osSrc_process_parent_userSid_to_sentinel_one_cloud_funnel_event_os_src_process_parent_user_sid_e402684c field: json.osSrc.process.parent.userSid target_field: sentinel_one_cloud_funnel.event.os_src_process.parent.user.sid ignore_missing: true - append: + tag: append_related_user_ffdaad55 field: related.user value: '{{{sentinel_one_cloud_funnel.event.os_src_process.parent.user.sid}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.os_src_process?.parent?.user?.sid != null - rename: + tag: rename_json_osSrc_process_userSid_to_sentinel_one_cloud_funnel_event_os_src_process_user_sid_03a6b37a field: json.osSrc.process.userSid target_field: sentinel_one_cloud_funnel.event.os_src_process.user.sid ignore_missing: true - append: + tag: append_related_user_1abe6994 field: related.user value: '{{{sentinel_one_cloud_funnel.event.os_src_process.user.sid}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.os_src_process?.user?.sid != null - rename: + tag: rename_json_registry_export_path_to_sentinel_one_cloud_funnel_event_registry_export_path_7f97d62d field: json.registry.export.path target_field: sentinel_one_cloud_funnel.event.registry.export_path ignore_missing: true - rename: + tag: rename_json_registry_import_path_to_sentinel_one_cloud_funnel_event_registry_import_path_e2da7c39 field: json.registry.import.path target_field: sentinel_one_cloud_funnel.event.registry.import_path ignore_missing: true - rename: + tag: rename_json_registry_owner_user_to_sentinel_one_cloud_funnel_event_registry_owner_user_name_1fe204e3 field: json.registry.owner.user target_field: sentinel_one_cloud_funnel.event.registry.owner.user.name ignore_missing: true - append: + tag: append_related_user_74d1f3ad field: related.user value: '{{{sentinel_one_cloud_funnel.event.registry.owner.user.name}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.registry?.owner?.user?.name != null - rename: + tag: rename_json_registry_owner_userSid_to_sentinel_one_cloud_funnel_event_registry_owner_user_sid_c3137356 field: json.registry.owner.userSid target_field: sentinel_one_cloud_funnel.event.registry.owner.user.sid ignore_missing: true - append: + tag: append_related_user_4aeaf4dd field: related.user value: '{{{sentinel_one_cloud_funnel.event.registry.owner.user.sid}}}' allow_duplicates: false @@ -2366,6 +2742,7 @@ processors: if: ctx.json?.registry?.security?.info != '' on_failure: - append: + tag: append_error_message_c1ee9dc6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2377,33 +2754,41 @@ processors: if: ctx.json?.src?.process?.exeModificationCount != '' on_failure: - append: + tag: append_error_message_3ddae393 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_src_process_image_description_to_sentinel_one_cloud_funnel_event_src_process_image_description_1ed88dee field: json.src.process.image.description target_field: sentinel_one_cloud_funnel.event.src.process.image.description ignore_missing: true - rename: + tag: rename_json_src_process_image_extension_to_sentinel_one_cloud_funnel_event_src_process_image_extension_d743d350 field: json.src.process.image.extension target_field: sentinel_one_cloud_funnel.event.src.process.image.extension ignore_missing: true - rename: + tag: rename_json_src_process_image_internalName_to_sentinel_one_cloud_funnel_event_src_process_image_internal_name_bf178241 field: json.src.process.image.internalName target_field: sentinel_one_cloud_funnel.event.src.process.image.internal_name ignore_missing: true - rename: + tag: rename_json_src_process_image_location_to_sentinel_one_cloud_funnel_event_src_process_image_location_2220bbc0 field: json.src.process.image.location target_field: sentinel_one_cloud_funnel.event.src.process.image.location ignore_missing: true - rename: + tag: rename_json_src_process_image_originalFileName_to_sentinel_one_cloud_funnel_event_src_process_image_original_file_name_0e5464a6 field: json.src.process.image.originalFileName target_field: sentinel_one_cloud_funnel.event.src.process.image.original_file_name ignore_missing: true - rename: + tag: rename_json_src_process_image_productName_to_sentinel_one_cloud_funnel_event_src_process_image_product_name_10598526 field: json.src.process.image.productName target_field: sentinel_one_cloud_funnel.event.src.process.image.product.name ignore_missing: true - rename: + tag: rename_json_src_process_image_productVersion_to_sentinel_one_cloud_funnel_event_src_process_image_product_version_44bff972 field: json.src.process.image.productVersion target_field: sentinel_one_cloud_funnel.event.src.process.image.product.version ignore_missing: true @@ -2416,13 +2801,16 @@ processors: if: ctx.json?.src?.process?.image?.size != '' on_failure: - append: + tag: append_error_message_cf981559 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_src_process_image_type_to_sentinel_one_cloud_funnel_event_src_process_image_type_693687c8 field: json.src.process.image.type target_field: sentinel_one_cloud_funnel.event.src.process.image.type ignore_missing: true - rename: + tag: rename_json_src_process_image_uid_to_sentinel_one_cloud_funnel_event_src_process_image_uid_f5d9231e field: json.src.process.image.uid target_field: sentinel_one_cloud_funnel.event.src.process.image.uid ignore_missing: true @@ -2435,6 +2823,7 @@ processors: if: ctx.json?.src?.process?.modelChildProcessCount != '' on_failure: - append: + tag: append_error_message_5096b1c0 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2446,13 +2835,16 @@ processors: if: ctx.json?.src?.process?.parent?.image?.binaryIsExecutable != '' on_failure: - append: + tag: append_error_message_5868d159 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_src_process_parent_image_extension_to_sentinel_one_cloud_funnel_event_src_process_parent_image_extension_3c074d2c field: json.src.process.parent.image.extension target_field: sentinel_one_cloud_funnel.event.src.process.parent.image.extension ignore_missing: true - rename: + tag: rename_json_src_process_parent_image_location_to_sentinel_one_cloud_funnel_event_src_process_parent_image_location_3d7298a8 field: json.src.process.parent.image.location target_field: sentinel_one_cloud_funnel.event.src.process.parent.image.location ignore_missing: true @@ -2465,6 +2857,7 @@ processors: if: ctx.json?.src?.process?.parent?.image?.signature?.isValid != '' on_failure: - append: + tag: append_error_message_50c01091 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2476,21 +2869,26 @@ processors: if: ctx.json?.src?.process?.parent?.image?.size != '' on_failure: - append: + tag: append_error_message_2f18484f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_src_process_parent_image_type_to_sentinel_one_cloud_funnel_event_src_process_parent_image_type_5fc8b4fa field: json.src.process.parent.image.type target_field: sentinel_one_cloud_funnel.event.src.process.parent.image.type ignore_missing: true - rename: + tag: rename_json_src_process_parent_image_uid_to_sentinel_one_cloud_funnel_event_src_process_parent_image_uid_bb289ea8 field: json.src.process.parent.image.uid target_field: sentinel_one_cloud_funnel.event.src.process.parent.image.uid ignore_missing: true - rename: + tag: rename_json_src_process_userSid_to_sentinel_one_cloud_funnel_event_src_process_user_sid_492102d8 field: json.src.process.userSid target_field: sentinel_one_cloud_funnel.event.src.process.user.sid ignore_missing: true - append: + tag: append_related_user_3da4603d field: related.user value: '{{{sentinel_one_cloud_funnel.event.src.process.user.sid}}}' allow_duplicates: false @@ -2504,6 +2902,7 @@ processors: if: ctx.json?.task?.triggerType != '' on_failure: - append: + tag: append_error_message_e40f4dfe field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2515,6 +2914,7 @@ processors: if: ctx.json?.tgt?.file?.isDirectory != '' on_failure: - append: + tag: append_error_message_05e729a4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -2526,34 +2926,42 @@ processors: if: ctx.json?.tgt?.file?.isKernelModule != '' on_failure: - append: + tag: append_error_message_45ea2e81 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tgt_file_originalFileName_to_sentinel_one_cloud_funnel_event_tgt_file_original_file_name_03990ff2 field: json.tgt.file.originalFileName target_field: sentinel_one_cloud_funnel.event.tgt.file.original_file_name ignore_missing: true - rename: + tag: rename_json_tgt_file_owner_name_to_sentinel_one_cloud_funnel_event_tgt_file_owner_name_caf1ecb8 field: json.tgt.file.owner.name target_field: sentinel_one_cloud_funnel.event.tgt.file.owner.name ignore_missing: true - rename: + tag: rename_json_tgt_file_owner_userSid_to_sentinel_one_cloud_funnel_event_tgt_file_owner_user_sid_80636bb9 field: json.tgt.file.owner.userSid target_field: sentinel_one_cloud_funnel.event.tgt.file.owner.user_sid ignore_missing: true - append: + tag: append_related_user_fa079c7b field: related.user value: '{{{sentinel_one_cloud_funnel.event.tgt.file.owner.user_sid}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.file?.owner?.user_sid != null - rename: + tag: rename_json_tgt_file_productName_to_sentinel_one_cloud_funnel_event_tgt_file_product_name_24dea404 field: json.tgt.file.productName target_field: sentinel_one_cloud_funnel.event.tgt.file.product.name ignore_missing: true - rename: + tag: rename_json_tgt_file_productVersion_to_sentinel_one_cloud_funnel_event_tgt_file_product_version_7d7fa924 field: json.tgt.file.productVersion target_field: sentinel_one_cloud_funnel.event.tgt.file.product.version ignore_missing: true - rename: + tag: rename_json_tgt_file_publisher_to_sentinel_one_cloud_funnel_event_tgt_file_publisher_3a6f1ed8 field: json.tgt.file.publisher target_field: sentinel_one_cloud_funnel.event.tgt.file.publisher ignore_missing: true @@ -2566,9 +2974,11 @@ processors: if: ctx.json?.tgt?.file?.signature?.isValid != '' on_failure: - append: + tag: append_error_message_a2405f9c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tgt_file_signatureInvalidReason_to_sentinel_one_cloud_funnel_event_tgt_file_signature_invalid_reason_70315ea1 field: json.tgt.file.signatureInvalidReason target_field: sentinel_one_cloud_funnel.event.tgt.file.signature.invalid_reason ignore_missing: true @@ -2581,9 +2991,11 @@ processors: if: ctx.json?.tgt?.process?.completeness?.hints != '' on_failure: - append: + tag: append_error_message_e925f864 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tgt_process_image_extension_to_sentinel_one_cloud_funnel_event_tgt_process_image_extension_e8621916 field: json.tgt.process.image.extension target_field: sentinel_one_cloud_funnel.event.tgt.process.image.extension ignore_missing: true @@ -2596,34 +3008,42 @@ processors: if: ctx.json?.tgt?.process?.image?.size != '' on_failure: - append: + tag: append_error_message_8994c5c1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tgt_process_image_uid_to_sentinel_one_cloud_funnel_event_tgt_process_image_uid_399089ac field: json.tgt.process.image.uid target_field: sentinel_one_cloud_funnel.event.tgt.process.image.uid ignore_missing: true - rename: + tag: rename_json_tgt_process_parent_image_location_to_sentinel_one_cloud_funnel_event_tgt_process_parent_image_location_2b271d56 field: json.tgt.process.parent.image.location target_field: sentinel_one_cloud_funnel.event.tgt.process.parent.image.location ignore_missing: true - rename: + tag: rename_json_tgt_process_parent_image_type_to_sentinel_one_cloud_funnel_event_tgt_process_parent_image_type_f5a5a810 field: json.tgt.process.parent.image.type target_field: sentinel_one_cloud_funnel.event.tgt.process.parent.image.type ignore_missing: true - rename: + tag: rename_json_tgt_process_userSid_to_sentinel_one_cloud_funnel_event_tgt_process_user_sid_959e1a22 field: json.tgt.process.userSid target_field: sentinel_one_cloud_funnel.event.tgt.process.user.sid ignore_missing: true - rename: + tag: rename_json_tgt_process_uid_to_sentinel_one_cloud_funnel_event_tgt_process_uid_d0b00836 field: json.tgt.process.uid target_field: sentinel_one_cloud_funnel.event.tgt.process.uid ignore_missing: true - append: + tag: append_related_user_07c6cfa3 field: related.user value: '{{{sentinel_one_cloud_funnel.event.tgt.process.user.sid}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.process?.user?.sid != null - rename: + tag: rename_json_tiIndicator_categories_to_sentinel_one_cloud_funnel_event_ti_indicator_categories_1aa2fb4d field: json.tiIndicator.categories target_field: sentinel_one_cloud_funnel.event.ti_indicator.categories ignore_missing: true @@ -2637,21 +3057,26 @@ processors: if: ctx.json?.tiIndicator?.creationTime != null && ctx.json.tiIndicator.creationTime != '' on_failure: - append: + tag: append_error_message_30209428 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tiIndicator_externalId_to_sentinel_one_cloud_funnel_event_ti_indicator_external_id_2b43cee2 field: json.tiIndicator.externalId target_field: sentinel_one_cloud_funnel.event.ti_indicator.external_id ignore_missing: true - rename: + tag: rename_json_tiIndicator_intrusionSets_to_sentinel_one_cloud_funnel_event_ti_indicator_intrusion_sets_14729c36 field: json.tiIndicator.intrusionSets target_field: sentinel_one_cloud_funnel.event.ti_indicator.intrusion_sets ignore_missing: true - rename: + tag: rename_json_tiIndicator_metadata_to_sentinel_one_cloud_funnel_event_ti_indicator_metadata_6f2b443b field: json.tiIndicator.metadata target_field: sentinel_one_cloud_funnel.event.ti_indicator.metadata ignore_missing: true - rename: + tag: rename_json_tiIndicator_threatActors_to_sentinel_one_cloud_funnel_event_ti_indicator_threat_actors_89d96830 field: json.tiIndicator.threatActors target_field: sentinel_one_cloud_funnel.event.ti_indicator.threat_actors ignore_missing: true @@ -2661,6 +3086,7 @@ processors: if: ctx.sentinel_one_cloud_funnel?.event?.category != null && ctx.sentinel_one_cloud_funnel.event.category.toLowerCase().contains('command') on_failure: - append: + tag: append_error_message_80a82c6e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - pipeline: @@ -2669,6 +3095,7 @@ processors: if: ctx.sentinel_one_cloud_funnel?.event?.category != null && ctx.sentinel_one_cloud_funnel.event.category.toLowerCase().contains('cross') on_failure: - append: + tag: append_error_message_c07c8943 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - pipeline: @@ -2677,6 +3104,7 @@ processors: if: ctx.sentinel_one_cloud_funnel?.event?.category != null && ctx.sentinel_one_cloud_funnel.event.category.toLowerCase().contains('dns') on_failure: - append: + tag: append_error_message_8560412e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - pipeline: @@ -2685,6 +3113,7 @@ processors: if: ctx.sentinel_one_cloud_funnel?.event?.category != null && ctx.sentinel_one_cloud_funnel.event.category.toLowerCase().contains('file') on_failure: - append: + tag: append_error_message_6e56ff53 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - pipeline: @@ -2693,6 +3122,7 @@ processors: if: ctx.sentinel_one_cloud_funnel?.event?.category != null && ctx.sentinel_one_cloud_funnel.event.category.toLowerCase().contains('indicator') && !(ctx.sentinel_one_cloud_funnel.event.category.toLowerCase().contains('threat')) on_failure: - append: + tag: append_error_message_232e2c88 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - pipeline: @@ -2701,6 +3131,7 @@ processors: if: ctx.sentinel_one_cloud_funnel?.event?.category != null && ctx.sentinel_one_cloud_funnel.event.category.toLowerCase().contains('login') on_failure: - append: + tag: append_error_message_8a60f0c8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - pipeline: @@ -2709,6 +3140,7 @@ processors: if: ctx.sentinel_one_cloud_funnel?.event?.category != null && ctx.sentinel_one_cloud_funnel.event.category.toLowerCase().contains('module') on_failure: - append: + tag: append_error_message_f149392d field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - pipeline: @@ -2717,6 +3149,7 @@ processors: if: ctx.sentinel_one_cloud_funnel?.event?.category != null && ctx.sentinel_one_cloud_funnel.event.category.toLowerCase().contains('ip') && !(ctx.sentinel_one_cloud_funnel.event.category.toLowerCase().contains('command')) on_failure: - append: + tag: append_error_message_c265e6dd field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - pipeline: @@ -2725,6 +3158,7 @@ processors: if: ctx.sentinel_one_cloud_funnel?.event?.category != null && ctx.sentinel_one_cloud_funnel.event.category.toLowerCase().contains('process') && !(ctx.sentinel_one_cloud_funnel.event.category.toLowerCase().contains('cross')) on_failure: - append: + tag: append_error_message_0c665c40 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - pipeline: @@ -2733,6 +3167,7 @@ processors: if: ctx.sentinel_one_cloud_funnel?.event?.category != null && ctx.sentinel_one_cloud_funnel.event.category.toLowerCase().contains('registry') on_failure: - append: + tag: append_error_message_b953bf4e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - pipeline: @@ -2741,6 +3176,7 @@ processors: if: ctx.sentinel_one_cloud_funnel?.event?.category != null && ctx.sentinel_one_cloud_funnel.event.category.toLowerCase().contains('schedule') on_failure: - append: + tag: append_error_message_bc229970 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - pipeline: @@ -2749,6 +3185,7 @@ processors: if: ctx.sentinel_one_cloud_funnel?.event?.category != null && ctx.sentinel_one_cloud_funnel.event.category.toLowerCase().contains('threat') on_failure: - append: + tag: append_error_message_898689d5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - pipeline: @@ -2757,9 +3194,11 @@ processors: if: ctx.sentinel_one_cloud_funnel?.event?.category != null && ctx.sentinel_one_cloud_funnel.event.category.toLowerCase().contains('url') on_failure: - append: + tag: append_error_message_772950ec field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: + tag: script_7c2a3c3b # This must happen after the pipelines since the process pipeline also touches process.command_line. description: Implements Windows-like SplitCommandLine lang: painless @@ -2854,9 +3293,11 @@ processors: ctx.process.parent.args_count = ctx.process.parent.args.length; } - remove: + tag: remove_json_29cdffdc field: json ignore_missing: true - remove: + tag: remove_9782d7f7 field: - sentinel_one_cloud_funnel.event.dst.ip_address - sentinel_one_cloud_funnel.event.dst.port_number @@ -2928,6 +3369,7 @@ processors: if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) ignore_missing: true - script: + tag: script_06971239 lang: painless description: Drops null/empty values recursively. source: |- @@ -2945,14 +3387,17 @@ processors: } drop(ctx); - set: + tag: set_event_kind_92954dfa field: event.kind value: pipeline_error if: ctx.error?.message != null - set: + tag: set_sentinel_one_cloud_funnel_event_rerouted_d02e579e field: sentinel_one_cloud_funnel.event.rerouted value: true if: ctx._conf?.reroute?.contains(ctx.sentinel_one_cloud_funnel?.event?.category?.toLowerCase()) == true - remove: + tag: remove__conf_8e93c735 field: _conf ignore_missing: true - set: @@ -2961,6 +3406,7 @@ processors: tag: set_pipeline_error_into_event_kind if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-command-script.yml b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-command-script.yml index 977fceadd3b..095418421c0 100644 --- a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-command-script.yml +++ b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-command-script.yml @@ -2,33 +2,41 @@ description: Pipeline for processing Command Script Event logs. processors: - set: + tag: set_event_category_dffe0229 field: event.category value: [process] - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] - rename: + tag: rename_json_tgt_file_sha1_to_sentinel_one_cloud_funnel_event_tgt_file_sha1_4685387e field: json.tgt.file.sha1 target_field: sentinel_one_cloud_funnel.event.tgt.file.sha1 ignore_missing: true - set: + tag: set_file_hash_sha1_5c6ba116 field: file.hash.sha1 copy_from: sentinel_one_cloud_funnel.event.tgt.file.sha1 ignore_empty_value: true - append: + tag: append_related_hash_b56c7961 field: related.hash value: '{{{file.hash.sha1}}}' allow_duplicates: false if: ctx.file?.hash?.sha1 != null - rename: + tag: rename_json_cmdScript_applicationName_to_sentinel_one_cloud_funnel_event_cmd_script_application_name_b1ca8d1a field: json.cmdScript.applicationName target_field: sentinel_one_cloud_funnel.event.cmd_script.application_name ignore_missing: true - rename: + tag: rename_json_cmdScript_content_to_sentinel_one_cloud_funnel_event_cmd_script_content_7f83794d field: json.cmdScript.content target_field: sentinel_one_cloud_funnel.event.cmd_script.content ignore_missing: true - set: + tag: set_powershell_file_script_block_text_82b728f0 field: powershell.file.script_block_text copy_from: sentinel_one_cloud_funnel.event.cmd_script.content ignore_empty_value: true @@ -41,6 +49,7 @@ processors: if: ctx.json?.cmdScript?.isComplete != '' on_failure: - append: + tag: append_error_message_29202d73 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -52,13 +61,16 @@ processors: if: ctx.json?.cmdScript?.originalSize != '' on_failure: - append: + tag: append_error_message_d7cc71af field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_cmdScript_sha256_to_sentinel_one_cloud_funnel_event_cmd_script_sha256_e27e61e5 field: json.cmdScript.sha256 target_field: sentinel_one_cloud_funnel.event.cmd_script.sha256 ignore_missing: true - append: + tag: append_related_hash_c7781308 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.cmd_script.sha256}}}' allow_duplicates: false @@ -72,9 +84,11 @@ processors: if: ctx.json?.src?.process != null && ctx.json.src.process['crossProcessOutOfStoryline™Count'] != '' on_failure: - append: + tag: append_error_message_a423316f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_src_process_isStoryline™Root_to_sentinel_one_cloud_funnel_event_src_process_is_storyline_tm_root_99123876 field: json.src.process.isStoryline™Root target_field: sentinel_one_cloud_funnel.event.src.process.is_storyline_tm_root ignore_missing: true @@ -87,13 +101,16 @@ processors: if: ctx.json?.src?.process?.parent != null && ctx.json.src.process.parent['isStoryline™Root'] != '' on_failure: - append: + tag: append_error_message_0f75dfb9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_src_process_parent_Storyline™_id_to_sentinel_one_cloud_funnel_event_src_process_parent_storyline_tm_id_9a709a5f field: json.src.process.parent.Storyline™.id target_field: sentinel_one_cloud_funnel.event.src.process.parent.storyline_tm_id ignore_missing: true - rename: + tag: rename_json_src_process_Storyline™_id_to_sentinel_one_cloud_funnel_event_src_process_storyline_tm_id_f075c2d7 field: json.src.process.Storyline™.id target_field: sentinel_one_cloud_funnel.event.src.process.storyline_tm_id ignore_missing: true @@ -104,3 +121,11 @@ on_failure: Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-cross-process.yml b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-cross-process.yml index 2fc4a23831c..0976b36e348 100644 --- a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-cross-process.yml +++ b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-cross-process.yml @@ -2,26 +2,32 @@ description: Pipeline for processing Cross Process Event logs. processors: - set: + tag: set_event_category_dffe0229 field: event.category value: [process] - set: + tag: set_event_type_ec95f7f2 field: event.type value: [info] - rename: + tag: rename_json_k8sCluster_containerImage_sha256_to_sentinel_one_cloud_funnel_event_k8s_cluster_container_image_sha256_997132df field: json.k8sCluster.containerImage.sha256 target_field: sentinel_one_cloud_funnel.event.k8s_cluster.container.image.sha256 ignore_missing: true - append: + tag: append_container_image_hash_all_7085001c field: container.image.hash.all value: '{{{sentinel_one_cloud_funnel.event.k8s_cluster.container.image.sha256}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.k8s_cluster?.container?.image?.sha256 != null - append: + tag: append_related_hash_fef1cce8 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.k8s_cluster.container.image.sha256}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.k8s_cluster?.container?.image?.sha256 != null - rename: + tag: rename_json_tgt_process_relation_to_sentinel_one_cloud_funnel_event_tgt_process_relation_3341485c field: json.tgt.process.relation target_field: sentinel_one_cloud_funnel.event.tgt.process.relation ignore_missing: true @@ -34,33 +40,41 @@ processors: if: ctx.json?.tgt?.process?.accessRights != '' on_failure: - append: + tag: append_error_message_658ceeba field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tgt_process_activeContent_hash_to_sentinel_one_cloud_funnel_event_tgt_process_active_content_hash_7e5be2bd field: json.tgt.process.activeContent.hash target_field: sentinel_one_cloud_funnel.event.tgt.process.active_content.hash ignore_missing: true - rename: + tag: rename_json_tgt_process_activeContent_id_to_sentinel_one_cloud_funnel_event_tgt_process_active_content_id_7796cb63 field: json.tgt.process.activeContent.id target_field: sentinel_one_cloud_funnel.event.tgt.process.active_content.id ignore_missing: true - rename: + tag: rename_json_tgt_process_activeContent_path_to_sentinel_one_cloud_funnel_event_tgt_process_active_content_path_c97e7e9f field: json.tgt.process.activeContent.path target_field: sentinel_one_cloud_funnel.event.tgt.process.active_content.path ignore_missing: true - rename: + tag: rename_json_tgt_process_activeContent_signedStatus_to_sentinel_one_cloud_funnel_event_tgt_process_active_content_signed_status_b1e15fcc field: json.tgt.process.activeContent.signedStatus target_field: sentinel_one_cloud_funnel.event.tgt.process.active_content.signed_status ignore_missing: true - rename: + tag: rename_json_tgt_process_activeContentType_to_sentinel_one_cloud_funnel_event_tgt_process_active_content_type_0e0027e7 field: json.tgt.process.activeContentType target_field: sentinel_one_cloud_funnel.event.tgt.process.active_content.type ignore_missing: true - rename: + tag: rename_json_tgt_process_cmdline_to_sentinel_one_cloud_funnel_event_tgt_process_cmd_line_812ccb5d field: json.tgt.process.cmdline target_field: sentinel_one_cloud_funnel.event.tgt.process.cmd_line ignore_missing: true - rename: + tag: rename_json_tgt_process_displayName_to_sentinel_one_cloud_funnel_event_tgt_process_display_name_abf63e69 field: json.tgt.process.displayName target_field: sentinel_one_cloud_funnel.event.tgt.process.display_name ignore_missing: true @@ -73,40 +87,49 @@ processors: if: ctx.json?.tgt?.process?.image?.binaryIsExecutable != '' on_failure: - append: + tag: append_error_message_16263fcf field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tgt_process_image_md5_to_sentinel_one_cloud_funnel_event_tgt_process_image_md5_edb32ef0 field: json.tgt.process.image.md5 target_field: sentinel_one_cloud_funnel.event.tgt.process.image.md5 ignore_missing: true - append: + tag: append_related_hash_e8d5a020 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.tgt.process.image.md5}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.process?.image?.md5 != null - rename: + tag: rename_json_tgt_process_image_path_to_sentinel_one_cloud_funnel_event_tgt_process_image_path_f507d268 field: json.tgt.process.image.path target_field: sentinel_one_cloud_funnel.event.tgt.process.image.path ignore_missing: true - rename: + tag: rename_json_tgt_process_image_sha1_to_sentinel_one_cloud_funnel_event_tgt_process_image_sha1_a2281558 field: json.tgt.process.image.sha1 target_field: sentinel_one_cloud_funnel.event.tgt.process.image.sha1 ignore_missing: true - append: + tag: append_related_hash_9e28d560 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.tgt.process.image.sha1}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.process?.image?.sha1 != null - rename: + tag: rename_json_tgt_process_image_sha256_to_sentinel_one_cloud_funnel_event_tgt_process_image_sha256_a4a15668 field: json.tgt.process.image.sha256 target_field: sentinel_one_cloud_funnel.event.tgt.process.image.sha256 ignore_missing: true - append: + tag: append_related_hash_a2061640 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.tgt.process.image.sha256}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.process?.image?.sha256 != null - rename: + tag: rename_json_tgt_process_integrityLevel_to_sentinel_one_cloud_funnel_event_tgt_process_integrity_level_1e2fbd25 field: json.tgt.process.integrityLevel target_field: sentinel_one_cloud_funnel.event.tgt.process.integrity_level ignore_missing: true @@ -119,6 +142,7 @@ processors: if: ctx.json?.tgt?.process?.isNative64Bit != '' on_failure: - append: + tag: append_error_message_7e1e78b8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -130,6 +154,7 @@ processors: if: ctx.json?.tgt?.process?.isRedirectCmdProcessor != '' on_failure: - append: + tag: append_error_message_5387e536 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -141,9 +166,11 @@ processors: if: ctx.json?.tgt?.process?.isStorylineRoot != '' on_failure: - append: + tag: append_error_message_bd000a95 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tgt_process_name_to_sentinel_one_cloud_funnel_event_tgt_process_name_27916ab4 field: json.tgt.process.name target_field: sentinel_one_cloud_funnel.event.tgt.process.name ignore_missing: true @@ -156,13 +183,16 @@ processors: if: ctx.json?.tgt?.process?.pid != '' on_failure: - append: + tag: append_error_message_1dd51f92 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tgt_process_publisher_to_sentinel_one_cloud_funnel_event_tgt_process_publisher_c8fa2f52 field: json.tgt.process.publisher target_field: sentinel_one_cloud_funnel.event.tgt.process.publisher ignore_missing: true - rename: + tag: rename_json_tgt_process_reasonSignatureInvalid_to_sentinel_one_cloud_funnel_event_tgt_process_reason_signature_invalid_b95abc7c field: json.tgt.process.reasonSignatureInvalid target_field: sentinel_one_cloud_funnel.event.tgt.process.reason_signature_invalid ignore_missing: true @@ -175,9 +205,11 @@ processors: if: ctx.json?.tgt?.process?.pid != '' on_failure: - append: + tag: append_error_message_8efa60f1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tgt_process_signedStatus_to_sentinel_one_cloud_funnel_event_tgt_process_signed_status_025c2f53 field: json.tgt.process.signedStatus target_field: sentinel_one_cloud_funnel.event.tgt.process.signed_status ignore_missing: true @@ -191,35 +223,43 @@ processors: if: ctx.json?.tgt?.process?.startTime != null && ctx.json.tgt.process.startTime != '' on_failure: - append: + tag: append_error_message_5fe39210 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tgt_process_storyline_id_to_sentinel_one_cloud_funnel_event_tgt_process_storyline_id_e1588021 field: json.tgt.process.storyline.id target_field: sentinel_one_cloud_funnel.event.tgt.process.storyline_id ignore_missing: true - rename: + tag: rename_json_tgt_process_subsystem_to_sentinel_one_cloud_funnel_event_tgt_process_subsystem_f2387370 field: json.tgt.process.subsystem target_field: sentinel_one_cloud_funnel.event.tgt.process.subsystem ignore_missing: true - rename: + tag: rename_json_tgt_process_uid_to_sentinel_one_cloud_funnel_event_tgt_process_uid_d0b00836 field: json.tgt.process.uid target_field: sentinel_one_cloud_funnel.event.tgt.process.uid ignore_missing: true - rename: + tag: rename_json_tgt_process_user_to_sentinel_one_cloud_funnel_event_tgt_process_user_name_b291e32f field: json.tgt.process.user target_field: sentinel_one_cloud_funnel.event.tgt.process.user.name ignore_missing: true - append: + tag: append_process_user_name_3d861cac field: process.user.name value: '{{{sentinel_one_cloud_funnel.event.tgt.process.user.name}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.process?.user?.name != null - append: + tag: append_related_user_72cd47f9 field: related.user value: '{{{sentinel_one_cloud_funnel.event.tgt.process.user.name}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.process?.user?.name != null - rename: + tag: rename_json_tgt_process_verifiedStatus_to_sentinel_one_cloud_funnel_event_tgt_process_verified_status_99559a2f field: json.tgt.process.verifiedStatus target_field: sentinel_one_cloud_funnel.event.tgt.process.verified_status ignore_missing: true @@ -229,6 +269,7 @@ processors: tag: set_pipeline_error_into_event_kind if: ctx.error?.message != null - append: + tag: append_tags_9fe66b2c field: tags value: preserve_original_event allow_duplicates: false @@ -240,3 +281,11 @@ on_failure: Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-dns.yml b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-dns.yml index 36becfbc335..9e802a85d72 100644 --- a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-dns.yml +++ b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-dns.yml @@ -2,19 +2,24 @@ description: Pipeline for processing DNS Event logs. processors: - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - set: + tag: set_event_type_7d9b55a0 field: event.type value: [info, protocol] - set: + tag: set_event_action_e66a1ba5 field: event.action value: [lookup_requested] - rename: + tag: rename_json_event_dns_request_to_sentinel_one_cloud_funnel_event_dns_request_4086bfdc field: json.event.dns.request target_field: sentinel_one_cloud_funnel.event.dns.request ignore_missing: true - rename: + tag: rename_json_event_dns_response_to_sentinel_one_cloud_funnel_event_dns_response_6302b082 field: json.event.dns.response target_field: sentinel_one_cloud_funnel.event.dns.response ignore_missing: true @@ -159,10 +164,12 @@ processors: } on_failure: - append: + tag: append_error_message_a04dea1e field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' allow_duplicates: false - foreach: + tag: foreach_dns_answers_3b0b739d field: dns.answers if: ctx.dns?.answers instanceof List ignore_failure: true @@ -172,6 +179,7 @@ processors: pattern: '^\[?::ffff:([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)(?:\](?::[0-9]+)?)?$' replacement: '$1' - foreach: + tag: foreach_dns_resolved_ip_0d1a1469 field: dns.resolved_ip if: ctx.dns?.resolved_ip instanceof List ignore_failure: true @@ -181,6 +189,7 @@ processors: pattern: '^\[?::ffff:([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)(?:\](?::[0-9]+)?)?$' replacement: '$1' - foreach: + tag: foreach_dns_resolved_ip_c85eb87d field: dns.resolved_ip if: ctx.dns?.resolved_ip instanceof List ignore_failure: true @@ -201,3 +210,11 @@ on_failure: Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-file.yml b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-file.yml index 5b92592f58d..1fe5ec017f8 100644 --- a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-file.yml +++ b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-file.yml @@ -2,17 +2,21 @@ description: Pipeline for processing File Event logs. processors: - set: + tag: set_event_category_3db32290 field: event.category value: [file] - set: + tag: set_event_type_90d42dbd field: event.type value: [creation] if: ctx.sentinel_one_cloud_funnel?.event?.type != null && ctx.sentinel_one_cloud_funnel.event.type.toLowerCase().contains('creat') - set: + tag: set_event_type_12d45eeb field: event.type value: [deletion] if: ctx.sentinel_one_cloud_funnel?.event?.type != null && ctx.sentinel_one_cloud_funnel.event.type.toLowerCase().contains('delet') - set: + tag: set_event_type_472bb559 field: event.type value: [change] if: >- @@ -23,18 +27,22 @@ processors: ctx.sentinel_one_cloud_funnel.event.type.toLowerCase().contains('rename') ) - set: + tag: set_event_type_f96f1f47 field: event.type value: [info] override: false - set: + tag: set_event_action_a46f2017 field: event.action value: [creation] if: ctx.sentinel_one_cloud_funnel?.event?.type != null && ctx.sentinel_one_cloud_funnel.event.type.toLowerCase().contains('creat') - set: + tag: set_event_action_2ff9593d field: event.action value: [deletion] if: ctx.sentinel_one_cloud_funnel?.event?.type != null && ctx.sentinel_one_cloud_funnel.event.type.toLowerCase().contains('delet') - set: + tag: set_event_action_9257a809 field: event.action value: [change] if: >- @@ -44,53 +52,65 @@ processors: ctx.sentinel_one_cloud_funnel.event.type.toLowerCase().contains('modif') ) - set: + tag: set_event_action_3db9cf47 field: event.action value: [rename] if: >- ctx.sentinel_one_cloud_funnel?.event?.type != null && ctx.sentinel_one_cloud_funnel.event.type.toLowerCase().contains('rename') - rename: + tag: rename_json_k8sCluster_containerId_to_sentinel_one_cloud_funnel_event_k8s_cluster_container_id_174e75ab field: json.k8sCluster.containerId target_field: sentinel_one_cloud_funnel.event.k8s_cluster.container.id ignore_missing: true - set: + tag: set_container_id_a11673c8 field: container.id copy_from: sentinel_one_cloud_funnel.event.k8s_cluster.container.id ignore_empty_value: true - rename: + tag: rename_json_k8sCluster_containerImage_sha256_to_sentinel_one_cloud_funnel_event_k8s_cluster_container_image_sha256_997132df field: json.k8sCluster.containerImage.sha256 target_field: sentinel_one_cloud_funnel.event.k8s_cluster.container.image.sha256 ignore_missing: true - append: + tag: append_container_image_hash_all_7085001c field: container.image.hash.all value: '{{{sentinel_one_cloud_funnel.event.k8s_cluster.container.image.sha256}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.k8s_cluster?.container?.image?.sha256 != null - append: + tag: append_related_hash_ca9c6fa4 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.k8s_cluster.container.image.sha256}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.k8s_cluster?.container?.image?.sha256!= null - rename: + tag: rename_json_k8sCluster_containerImage_value_to_sentinel_one_cloud_funnel_event_k8s_cluster_container_image_value_4e464a3d field: json.k8sCluster.containerImage.value target_field: sentinel_one_cloud_funnel.event.k8s_cluster.container.image.value ignore_missing: true - set: + tag: set_container_image_name_6b959730 field: container.image.name copy_from: sentinel_one_cloud_funnel.event.k8s_cluster.container.image.value ignore_empty_value: true - rename: + tag: rename_json_k8sCluster_containerLabels_to_sentinel_one_cloud_funnel_event_k8s_cluster_container_labels_3938f7d7 field: json.k8sCluster.containerLabels target_field: sentinel_one_cloud_funnel.event.k8s_cluster.container.labels ignore_missing: true - set: + tag: set_container_labels_9d053fd0 field: container.labels copy_from: sentinel_one_cloud_funnel.event.k8s_cluster.container.labels ignore_empty_value: true - rename: + tag: rename_json_k8sCluster_containerName_to_sentinel_one_cloud_funnel_event_k8s_cluster_container_name_ee0d5ad3 field: json.k8sCluster.containerName target_field: sentinel_one_cloud_funnel.event.k8s_cluster.container.name ignore_missing: true - set: + tag: set_container_name_979f1054 field: container.name copy_from: sentinel_one_cloud_funnel.event.k8s_cluster.container.name ignore_empty_value: true @@ -104,55 +124,68 @@ processors: if: ctx.json?.tgt?.file?.creationTime != null && ctx.json.tgt.file.creationTime != '' on_failure: - append: + tag: append_error_message_3a00d0dc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_file_created_d1ec61e1 field: file.created copy_from: sentinel_one_cloud_funnel.event.tgt.file.creation_time ignore_empty_value: true - rename: + tag: rename_json_tgt_file_extension_to_sentinel_one_cloud_funnel_event_tgt_file_extension_0c7969fc field: json.tgt.file.extension target_field: sentinel_one_cloud_funnel.event.tgt.file.extension ignore_missing: true - set: + tag: set_file_extension_e5c5c6d4 field: file.extension copy_from: sentinel_one_cloud_funnel.event.tgt.file.extension ignore_empty_value: true - rename: + tag: rename_json_tgt_file_md5_to_sentinel_one_cloud_funnel_event_tgt_file_md5_f59548a0 field: json.tgt.file.md5 target_field: sentinel_one_cloud_funnel.event.tgt.file.md5 ignore_missing: true - set: + tag: set_file_hash_md5_b5abcf60 field: file.hash.md5 copy_from: sentinel_one_cloud_funnel.event.tgt.file.md5 ignore_empty_value: true - append: + tag: append_related_hash_6af42da3 field: related.hash value: '{{{file.hash.md5}}}' allow_duplicates: false if: ctx.file?.hash?.md5 != null - rename: + tag: rename_json_tgt_file_sha1_to_sentinel_one_cloud_funnel_event_tgt_file_sha1_4685387e field: json.tgt.file.sha1 target_field: sentinel_one_cloud_funnel.event.tgt.file.sha1 ignore_missing: true - set: + tag: set_file_hash_sha1_5c6ba116 field: file.hash.sha1 copy_from: sentinel_one_cloud_funnel.event.tgt.file.sha1 ignore_empty_value: true - append: + tag: append_related_hash_b56c7961 field: related.hash value: '{{{file.hash.sha1}}}' allow_duplicates: false if: ctx.file?.hash?.sha1 != null - rename: + tag: rename_json_tgt_file_sha256_to_sentinel_one_cloud_funnel_event_tgt_file_sha256_94584cca field: json.tgt.file.sha256 target_field: sentinel_one_cloud_funnel.event.tgt.file.sha256 ignore_missing: true - set: + tag: set_file_hash_sha256_df14789e field: file.hash.sha256 copy_from: sentinel_one_cloud_funnel.event.tgt.file.sha256 ignore_empty_value: true - append: + tag: append_related_hash_017aef51 field: related.hash value: '{{{file.hash.sha256}}}' allow_duplicates: false @@ -167,29 +200,36 @@ processors: if: ctx.json?.tgt?.file?.modificationTime != null && ctx.json.tgt.file.modificationTime != '' on_failure: - append: + tag: append_error_message_d1a23c4b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_file_mtime_88ff3fe4 field: file.mtime copy_from: sentinel_one_cloud_funnel.event.tgt.file.modification_time ignore_empty_value: true - rename: + tag: rename_json_tgt_file_path_to_sentinel_one_cloud_funnel_event_tgt_file_path_39609c16 field: json.tgt.file.path target_field: sentinel_one_cloud_funnel.event.tgt.file.path ignore_missing: true - set: + tag: set_file_path_527bee9a field: file.path copy_from: sentinel_one_cloud_funnel.event.tgt.file.path ignore_empty_value: true - rename: + tag: rename_json_tgt_file_type_to_sentinel_one_cloud_funnel_event_tgt_file_type_d8855d28 field: json.tgt.file.type target_field: sentinel_one_cloud_funnel.event.tgt.file.type ignore_missing: true - set: + tag: set_file_type_6b11f406 field: file.type copy_from: sentinel_one_cloud_funnel.event.tgt.file.type ignore_empty_value: true - script: + tag: script_81415464 lang: painless if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.file?.path instanceof String && ctx.sentinel_one_cloud_funnel.event.tgt.file.path.length() > 1 source: |- @@ -222,9 +262,11 @@ processors: if: ctx.json?.tgt?.file?.size != '' on_failure: - append: + tag: append_error_message_96037225 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_file_size_53ce742a field: file.size copy_from: sentinel_one_cloud_funnel.event.tgt.file.size ignore_empty_value: true @@ -237,84 +279,104 @@ processors: if: ctx.json?.src?.process?.tid != '' on_failure: - append: + tag: append_error_message_114b3353 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_process_thread_id_95a0158f field: process.thread.id copy_from: sentinel_one_cloud_funnel.event.src.process.tid ignore_empty_value: true - rename: + tag: rename_json_tgt_file_oldMd5_to_sentinel_one_cloud_funnel_event_tgt_file_old_md5_790f5fe0 field: json.tgt.file.oldMd5 target_field: sentinel_one_cloud_funnel.event.tgt.file.old.md5 ignore_missing: true - append: + tag: append_related_hash_2eec9328 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.tgt.file.old.md5}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.file?.old?.md5 != null - rename: + tag: rename_json_tgt_file_oldPath_to_sentinel_one_cloud_funnel_event_tgt_file_old_path_24dce2f0 field: json.tgt.file.oldPath target_field: sentinel_one_cloud_funnel.event.tgt.file.old.path ignore_missing: true - rename: + tag: rename_json_tgt_file_oldSha1_to_sentinel_one_cloud_funnel_event_tgt_file_old_sha1_13e16564 field: json.tgt.file.oldSha1 target_field: sentinel_one_cloud_funnel.event.tgt.file.old.sha1 ignore_missing: true - append: + tag: append_related_hash_92bf0db6 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.tgt.file.old.sha1}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.file?.old?.sha1!= null - rename: + tag: rename_json_tgt_file_oldSha256_to_sentinel_one_cloud_funnel_event_tgt_file_old_sha256_3bbb7b0c field: json.tgt.file.oldSha256 target_field: sentinel_one_cloud_funnel.event.tgt.file.old.sha256 ignore_missing: true - append: + tag: append_related_hash_82eebe90 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.tgt.file.old.sha256}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.file?.old?.sha256 != null - rename: + tag: rename_json_k8sCluster_controllerLabels_to_sentinel_one_cloud_funnel_event_k8s_cluster_controller_labels_10e1beab field: json.k8sCluster.controllerLabels target_field: sentinel_one_cloud_funnel.event.k8s_cluster.controller.labels ignore_missing: true - rename: + tag: rename_json_k8sCluster_controllerName_to_sentinel_one_cloud_funnel_event_k8s_cluster_controller_name_f2dcde6f field: json.k8sCluster.controllerName target_field: sentinel_one_cloud_funnel.event.k8s_cluster.controller.name ignore_missing: true - rename: + tag: rename_json_k8sCluster_controllerType_to_sentinel_one_cloud_funnel_event_k8s_cluster_controller_type_985eff4d field: json.k8sCluster.controllerType target_field: sentinel_one_cloud_funnel.event.k8s_cluster.controller.type ignore_missing: true - rename: + tag: rename_json_k8sCluster_name_to_sentinel_one_cloud_funnel_event_k8s_cluster_name_e94cd745 field: json.k8sCluster.name target_field: sentinel_one_cloud_funnel.event.k8s_cluster.name ignore_missing: true - rename: + tag: rename_json_k8sCluster_namespace_to_sentinel_one_cloud_funnel_event_k8s_cluster_namespace_value_cb9b4996 field: json.k8sCluster.namespace target_field: sentinel_one_cloud_funnel.event.k8s_cluster.namespace.value ignore_missing: true - rename: + tag: rename_json_k8sCluster_namespaceLabels_to_sentinel_one_cloud_funnel_event_k8s_cluster_namespace_labels_77ed3adf field: json.k8sCluster.namespaceLabels target_field: sentinel_one_cloud_funnel.event.k8s_cluster.namespace.labels ignore_missing: true - rename: + tag: rename_json_k8sCluster_nodeName_to_sentinel_one_cloud_funnel_event_k8s_cluster_node_name_494c1d46 field: json.k8sCluster.nodeName target_field: sentinel_one_cloud_funnel.event.k8s_cluster.node_name ignore_missing: true - rename: + tag: rename_json_k8sCluster_podLabels_to_sentinel_one_cloud_funnel_event_k8s_cluster_pod_labels_d3757fbf field: json.k8sCluster.podLabels target_field: sentinel_one_cloud_funnel.event.k8s_cluster.pod.labels ignore_missing: true - rename: + tag: rename_json_k8sCluster_podName_to_sentinel_one_cloud_funnel_event_k8s_cluster_pod_name_5bdc55a3 field: json.k8sCluster.podName target_field: sentinel_one_cloud_funnel.event.k8s_cluster.pod.name ignore_missing: true - rename: + tag: rename_json_src_process_reasonSignatureInvalid_to_sentinel_one_cloud_funnel_event_src_process_reason_signature_invalid_8220e978 field: json.src.process.reasonSignatureInvalid target_field: sentinel_one_cloud_funnel.event.src.process.reason_signature_invalid ignore_missing: true - convert: + tag: convert_json_src_process_rpid_to_sentinel_one_cloud_funnel_event_src_process_rpid_422ec69e field: json.src.process.rpid target_field: sentinel_one_cloud_funnel.event.src.process.rpid type: string @@ -322,25 +384,31 @@ processors: if: ctx.json?.src?.process?.rpid != '' on_failure: - append: + tag: append_error_message_1afff369 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_task_path_to_sentinel_one_cloud_funnel_event_task_path_a1b7920a field: json.task.path target_field: sentinel_one_cloud_funnel.event.task.path ignore_missing: true - rename: + tag: rename_json_tgt_file_convictedBy_to_sentinel_one_cloud_funnel_event_tgt_file_convicted_by_9dae0cc9 field: json.tgt.file.convictedBy target_field: sentinel_one_cloud_funnel.event.tgt.file.convicted_by ignore_missing: true - rename: + tag: rename_json_tgt_file_description_to_sentinel_one_cloud_funnel_event_tgt_file_description_54771acc field: json.tgt.file.description target_field: sentinel_one_cloud_funnel.event.tgt.file.description ignore_missing: true - rename: + tag: rename_json_tgt_file_id_to_sentinel_one_cloud_funnel_event_tgt_file_id_76a97c5e field: json.tgt.file.id target_field: sentinel_one_cloud_funnel.event.tgt.file.id ignore_missing: true - rename: + tag: rename_json_tgt_file_internalName_to_sentinel_one_cloud_funnel_event_tgt_file_internal_name_84e0ac81 field: json.tgt.file.internalName target_field: sentinel_one_cloud_funnel.event.tgt.file.internal_name ignore_missing: true @@ -353,9 +421,11 @@ processors: if: ctx.json?.tgt?.file?.isExecutable != '' on_failure: - append: + tag: append_error_message_7406da9a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tgt_file_location_to_sentinel_one_cloud_funnel_event_tgt_file_location_90526cb6 field: json.tgt.file.location target_field: sentinel_one_cloud_funnel.event.tgt.file.location ignore_missing: true @@ -366,3 +436,11 @@ on_failure: Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-indicator.yml b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-indicator.yml index 1c34ae785d5..95ffc99b798 100644 --- a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-indicator.yml +++ b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-indicator.yml @@ -10,25 +10,31 @@ processors: if: ctx.json?.src?.process?.tid != '' on_failure: - append: + tag: append_error_message_114b3353 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_process_thread_id_95a0158f field: process.thread.id copy_from: sentinel_one_cloud_funnel.event.src.process.tid ignore_empty_value: true - rename: + tag: rename_json_indicator_category_to_sentinel_one_cloud_funnel_event_indicator_category_cb846a94 field: json.indicator.category target_field: sentinel_one_cloud_funnel.event.indicator.category ignore_missing: true - rename: + tag: rename_json_indicator_description_to_sentinel_one_cloud_funnel_event_indicator_description_2c9c5a20 field: json.indicator.description target_field: sentinel_one_cloud_funnel.event.indicator.description ignore_missing: true - rename: + tag: rename_json_indicator_metadata_to_sentinel_one_cloud_funnel_event_indicator_metadata_2189b464 field: json.indicator.metadata target_field: sentinel_one_cloud_funnel.event.indicator.metadata ignore_missing: true - rename: + tag: rename_json_indicator_name_to_sentinel_one_cloud_funnel_event_indicator_name_d837b6a0 field: json.indicator.name target_field: sentinel_one_cloud_funnel.event.indicator.name ignore_missing: true @@ -41,6 +47,7 @@ processors: if: ctx.json?.src?.process != null && ctx.json.src.process['isStoryline™Root'] != '' on_failure: - append: + tag: append_error_message_4efa3827 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -52,13 +59,16 @@ processors: if: ctx.json?.src?.process?.parent != null && ctx.json.src.process.parent['isStoryline™Root'] != '' on_failure: - append: + tag: append_error_message_88005829 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_src_process_parent_Storyline™_id_to_sentinel_one_cloud_funnel_event_src_process_parent_storyline_tm_id_9a709a5f field: json.src.process.parent.Storyline™.id target_field: sentinel_one_cloud_funnel.event.src.process.parent.storyline_tm_id ignore_missing: true - rename: + tag: rename_json_src_process_Storyline™_id_to_sentinel_one_cloud_funnel_event_src_process_storyline_tm_id_f075c2d7 field: json.src.process.Storyline™.id target_field: sentinel_one_cloud_funnel.event.src.process.storyline_tm_id ignore_missing: true @@ -69,3 +79,11 @@ on_failure: Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-login.yml b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-login.yml index 2f6c9a7c474..72f3f69c2af 100644 --- a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-login.yml +++ b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-login.yml @@ -2,29 +2,36 @@ description: Pipeline for processing Login Event logs. processors: - set: + tag: set_event_category_e43d8992 field: event.category value: [authentication] - set: + tag: set_event_type_3752c517 field: event.type value: [start] if: ctx.sentinel_one_cloud_funnel?.event?.type == 'Login' - set: + tag: set_event_type_14da41ed field: event.type value: [end] if: ctx.sentinel_one_cloud_funnel?.event?.type == 'Logout' - set: + tag: set_event_type_f96f1f47 field: event.type value: [info] override: false - rename: + tag: rename_json_event_login_userName_to_sentinel_one_cloud_funnel_event_login_user_name_9c1af717 field: json.event.login.userName target_field: sentinel_one_cloud_funnel.event.login.user_name ignore_missing: true - set: + tag: set_user_name_e2f9cbbd field: user.name copy_from: sentinel_one_cloud_funnel.event.login.user_name ignore_empty_value: true - append: + tag: append_related_user_837e080f field: related.user value: '{{{user.name}}}' allow_duplicates: false @@ -38,40 +45,49 @@ processors: if: ctx.json?.src?.endpoint?.ip?.address != '' on_failure: - append: + tag: append_error_message_436aaa5c field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: + tag: append_related_ip_2bb46597 field: related.ip value: '{{{sentinel_one_cloud_funnel.event.src.endpoint_ip_address}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.src?.endpoint_ip_address != null - append: + tag: append_host_ip_0a2e6a3e field: host.ip value: '{{{sentinel_one_cloud_funnel.event.src.endpoint_ip_address}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.src?.endpoint_ip_address != null - append: + tag: append_related_hosts_80060767 field: related.hosts value: '{{{sentinel_one_cloud_funnel.event.src.endpoint_ip_address}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.src?.endpoint_ip_address != null - rename: + tag: rename_json_event_login_accountDomain_to_sentinel_one_cloud_funnel_event_login_account_domain_429768e2 field: json.event.login.accountDomain target_field: sentinel_one_cloud_funnel.event.login.account.domain ignore_missing: true - rename: + tag: rename_json_event_login_accountName_to_sentinel_one_cloud_funnel_event_login_account_name_bff34912 field: json.event.login.accountName target_field: sentinel_one_cloud_funnel.event.login.account.name ignore_missing: true - rename: + tag: rename_json_event_login_accountSid_to_sentinel_one_cloud_funnel_event_login_account_sid_c41685d8 field: json.event.login.accountSid target_field: sentinel_one_cloud_funnel.event.login.account.sid ignore_missing: true - rename: + tag: rename_json_event_login_baseType_to_sentinel_one_cloud_funnel_event_login_base_type_33e5ba79 field: json.event.login.baseType target_field: sentinel_one_cloud_funnel.event.login.base_type ignore_missing: true - rename: + tag: rename_json_event_login_failureReason_to_sentinel_one_cloud_funnel_event_login_failure_reason_dbb7daa1 field: json.event.login.failureReason target_field: sentinel_one_cloud_funnel.event.login.failure_reason ignore_missing: true @@ -84,6 +100,7 @@ processors: if: ctx.json?.event?.login?.isAdministratorEquivalent != '' on_failure: - append: + tag: append_error_message_724a8d6b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -95,6 +112,7 @@ processors: if: ctx.json?.event?.login?.loginIsSuccessful != '' on_failure: - append: + tag: append_error_message_31499749 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -106,9 +124,11 @@ processors: if: ctx.json?.event?.login?.sessionId != '' on_failure: - append: + tag: append_error_message_9cd484f7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_event_login_type_to_sentinel_one_cloud_funnel_event_login_type_b4089f96 field: json.event.login.type target_field: sentinel_one_cloud_funnel.event.login.type ignore_missing: true @@ -119,3 +139,11 @@ on_failure: Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-module.yml b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-module.yml index 67aba414351..731953aadea 100644 --- a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-module.yml +++ b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-module.yml @@ -2,23 +2,28 @@ description: Pipeline for processing Module Event logs. processors: - rename: + tag: rename_json_module_md5_to_sentinel_one_cloud_funnel_event_module_md5_a1cc2140 field: json.module.md5 target_field: sentinel_one_cloud_funnel.event.module.md5 ignore_missing: true - append: + tag: append_related_hash_56e48f64 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.module.md5}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.module?.md5 != null - rename: + tag: rename_json_module_path_to_sentinel_one_cloud_funnel_event_module_path_09423480 field: json.module.path target_field: sentinel_one_cloud_funnel.event.module.path ignore_missing: true - rename: + tag: rename_json_module_sha1_to_sentinel_one_cloud_funnel_event_module_sha1_bc0e7c88 field: json.module.sha1 target_field: sentinel_one_cloud_funnel.event.module.sha1 ignore_missing: true - append: + tag: append_related_hash_a56fc778 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.module.sha1}}}' allow_duplicates: false @@ -30,3 +35,11 @@ on_failure: Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-network-action.yml b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-network-action.yml index 3a4cf1cdf4f..b1556d8e2fa 100644 --- a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-network-action.yml +++ b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-network-action.yml @@ -2,65 +2,78 @@ description: Pipeline for processing Network Action Event logs. processors: - set: + tag: set_event_category_dbab8a4e field: event.category value: [network] - set: + tag: set_event_type_0e755d2c field: event.type value: [start, connection] if: (ctx.sentinel_one_cloud_funnel?.event?.type == 'IPConnect' || ctx.sentinel_one_cloud_funnel?.event?.type == 'IP Connect') - set: + tag: set_event_action_0154f3ad field: event.action value: [connection_attempted] - if: (ctx.sentinel_one_cloud_funnel?.event?.type == 'IPConnect' || ctx.sentinel_one_cloud_funnel?.event?.type == 'IP Connect') && - ctx.json.event?.network?.direction == "OUTGOING" + if: (ctx.sentinel_one_cloud_funnel?.event?.type == 'IPConnect' || ctx.sentinel_one_cloud_funnel?.event?.type == 'IP Connect') && ctx.json.event?.network?.direction == "OUTGOING" - set: + tag: set_event_action_042b17ce field: event.action value: [connection_accepted] - if: (ctx.sentinel_one_cloud_funnel?.event?.type == 'IPConnect' || ctx.sentinel_one_cloud_funnel?.event?.type == 'IP Connect') && - ctx.json.event?.network?.direction == "INCOMING" + if: (ctx.sentinel_one_cloud_funnel?.event?.type == 'IPConnect' || ctx.sentinel_one_cloud_funnel?.event?.type == 'IP Connect') && ctx.json.event?.network?.direction == "INCOMING" - rename: + tag: rename_json_k8sCluster_containerId_to_sentinel_one_cloud_funnel_event_k8s_cluster_container_id_174e75ab field: json.k8sCluster.containerId target_field: sentinel_one_cloud_funnel.event.k8s_cluster.container.id ignore_missing: true - set: + tag: set_container_id_a11673c8 field: container.id copy_from: sentinel_one_cloud_funnel.event.k8s_cluster.container.id ignore_empty_value: true - rename: + tag: rename_json_k8sCluster_containerImage_sha256_to_sentinel_one_cloud_funnel_event_k8s_cluster_container_image_sha256_997132df field: json.k8sCluster.containerImage.sha256 target_field: sentinel_one_cloud_funnel.event.k8s_cluster.container.image.sha256 ignore_missing: true - append: + tag: append_container_image_hash_all_7085001c field: container.image.hash.all value: '{{{sentinel_one_cloud_funnel.event.k8s_cluster.container.image.sha256}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.k8s_cluster?.container?.image?.sha256 != null - append: + tag: append_related_hash_fef1cce8 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.k8s_cluster.container.image.sha256}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.k8s_cluster?.container?.image?.sha256 != null - rename: + tag: rename_json_k8sCluster_containerImage_value_to_sentinel_one_cloud_funnel_event_k8s_cluster_container_image_value_4e464a3d field: json.k8sCluster.containerImage.value target_field: sentinel_one_cloud_funnel.event.k8s_cluster.container.image.value ignore_missing: true - set: + tag: set_container_image_name_6b959730 field: container.image.name copy_from: sentinel_one_cloud_funnel.event.k8s_cluster.container.image.value ignore_empty_value: true - rename: + tag: rename_json_k8sCluster_containerLabels_to_sentinel_one_cloud_funnel_event_k8s_cluster_container_labels_3938f7d7 field: json.k8sCluster.containerLabels target_field: sentinel_one_cloud_funnel.event.k8s_cluster.container.labels ignore_missing: true - set: + tag: set_container_labels_9d053fd0 field: container.labels copy_from: sentinel_one_cloud_funnel.event.k8s_cluster.container.labels ignore_empty_value: true - rename: + tag: rename_json_k8sCluster_containerName_to_sentinel_one_cloud_funnel_event_k8s_cluster_container_name_ee0d5ad3 field: json.k8sCluster.containerName target_field: sentinel_one_cloud_funnel.event.k8s_cluster.container.name ignore_missing: true - set: + tag: set_container_name_979f1054 field: container.name copy_from: sentinel_one_cloud_funnel.event.k8s_cluster.container.name ignore_empty_value: true @@ -73,18 +86,22 @@ processors: if: ctx.json?.dst?.ip?.address != '' on_failure: - append: + tag: append_error_message_1277991f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: + tag: append_related_ip_df2792db field: related.ip value: '{{{sentinel_one_cloud_funnel.event.dst.ip_address}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.dst?.ip_address != null - set: + tag: set_destination_ip_a07c80e5 field: destination.ip copy_from: sentinel_one_cloud_funnel.event.dst.ip_address ignore_empty_value: true - set: + tag: set_destination_address_cb11b966 field: destination.address copy_from: sentinel_one_cloud_funnel.event.dst.ip_address ignore_empty_value: true @@ -97,17 +114,21 @@ processors: if: ctx.json?.dst?.port?.number != '' on_failure: - append: + tag: append_error_message_35e897a2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_destination_port_ebf48f1c field: destination.port copy_from: sentinel_one_cloud_funnel.event.dst.port_number ignore_empty_value: true - rename: + tag: rename_json_event_network_connectionStatus_to_sentinel_one_cloud_funnel_event_network_connection_status_b98a8e91 field: json.event.network.connectionStatus target_field: sentinel_one_cloud_funnel.event.network.connection_status ignore_missing: true - set: + tag: set_event_outcome_c108e09d field: event.outcome value: 'unknown' if: ctx.sentinel_one_cloud_funnel?.event?.network?.connection_status != null && ctx.sentinel_one_cloud_funnel.event.network.connection_status.toLowerCase() == ('blocked') @@ -119,21 +140,26 @@ processors: if: ctx.sentinel_one_cloud_funnel?.event?.network?.connection_status != '' && ctx.event?.outcome != ('unknown') on_failure: - append: + tag: append_error_message_fab6111f field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_event_network_direction_to_sentinel_one_cloud_funnel_event_network_direction_a0b8f5b6 field: json.event.network.direction target_field: sentinel_one_cloud_funnel.event.network.direction ignore_missing: true - set: + tag: set_network_direction_87ae7381 field: network.direction value: 'ingress' if: ctx.sentinel_one_cloud_funnel?.event?.network?.direction != null && ctx.sentinel_one_cloud_funnel.event.network.direction.toLowerCase() == ('incoming') - set: + tag: set_network_direction_b44daa83 field: network.direction value: 'egress' if: ctx.sentinel_one_cloud_funnel?.event?.network?.direction != null && ctx.sentinel_one_cloud_funnel.event.network.direction.toLowerCase() == ('outgoing') - rename: + tag: rename_json_event_network_protocolName_to_sentinel_one_cloud_funnel_event_network_protocol_name_f5e572c3 field: json.event.network.protocolName target_field: sentinel_one_cloud_funnel.event.network.protocol_name ignore_missing: true @@ -145,6 +171,7 @@ processors: if: ctx.sentinel_one_cloud_funnel?.event?.network?.protocol_name != '' on_failure: - append: + tag: append_error_message_8a883275 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -156,14 +183,17 @@ processors: if: ctx.json?.src?.ip?.address != '' on_failure: - append: + tag: append_error_message_c11a6e23 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: + tag: append_related_ip_ca26f38d field: related.ip value: '{{{sentinel_one_cloud_funnel.event.src.ip_address}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.src?.ip?.address != null - set: + tag: set_source_ip_d21eb9f1 field: source.ip copy_from: sentinel_one_cloud_funnel.event.src.ip_address ignore_empty_value: true @@ -176,49 +206,61 @@ processors: if: ctx.json?.src?.port?.number != '' on_failure: - append: + tag: append_error_message_c7ab68ca field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_source_port_a2c399d4 field: source.port copy_from: sentinel_one_cloud_funnel.event.src.port_number ignore_empty_value: true - rename: + tag: rename_json_k8sCluster_controllerLabels_to_sentinel_one_cloud_funnel_event_k8s_cluster_controller_labels_10e1beab field: json.k8sCluster.controllerLabels target_field: sentinel_one_cloud_funnel.event.k8s_cluster.controller.labels ignore_missing: true - rename: + tag: rename_json_k8sCluster_controllerName_to_sentinel_one_cloud_funnel_event_k8s_cluster_controller_name_f2dcde6f field: json.k8sCluster.controllerName target_field: sentinel_one_cloud_funnel.event.k8s_cluster.controller.name ignore_missing: true - rename: + tag: rename_json_k8sCluster_controllerType_to_sentinel_one_cloud_funnel_event_k8s_cluster_controller_type_985eff4d field: json.k8sCluster.controllerType target_field: sentinel_one_cloud_funnel.event.k8s_cluster.controller.type ignore_missing: true - rename: + tag: rename_json_k8sCluster_name_to_sentinel_one_cloud_funnel_event_k8s_cluster_name_e94cd745 field: json.k8sCluster.name target_field: sentinel_one_cloud_funnel.event.k8s_cluster.name ignore_missing: true - rename: + tag: rename_json_k8sCluster_namespace_to_sentinel_one_cloud_funnel_event_k8s_cluster_namespace_value_cb9b4996 field: json.k8sCluster.namespace target_field: sentinel_one_cloud_funnel.event.k8s_cluster.namespace.value ignore_missing: true - rename: + tag: rename_json_k8sCluster_namespaceLabels_to_sentinel_one_cloud_funnel_event_k8s_cluster_namespace_labels_77ed3adf field: json.k8sCluster.namespaceLabels target_field: sentinel_one_cloud_funnel.event.k8s_cluster.namespace.labels ignore_missing: true - rename: + tag: rename_json_k8sCluster_nodeName_to_sentinel_one_cloud_funnel_event_k8s_cluster_node_name_494c1d46 field: json.k8sCluster.nodeName target_field: sentinel_one_cloud_funnel.event.k8s_cluster.node_name ignore_missing: true - rename: + tag: rename_json_k8sCluster_podLabels_to_sentinel_one_cloud_funnel_event_k8s_cluster_pod_labels_d3757fbf field: json.k8sCluster.podLabels target_field: sentinel_one_cloud_funnel.event.k8s_cluster.pod.labels ignore_missing: true - rename: + tag: rename_json_k8sCluster_podName_to_sentinel_one_cloud_funnel_event_k8s_cluster_pod_name_5bdc55a3 field: json.k8sCluster.podName target_field: sentinel_one_cloud_funnel.event.k8s_cluster.pod.name ignore_missing: true - append: + tag: append_event_type_c0894b0f field: event.type value: connection allow_duplicates: false @@ -226,6 +268,7 @@ processors: (ctx.source?.ip != null || ctx.source?.address != null) && ctx.source?.port != null && (ctx.destination?.ip != null || ctx.destination?.address != null) && ctx.destination?.port != null - set: + tag: set_event_type_f96f1f47 field: event.type value: [info] override: false @@ -236,3 +279,11 @@ on_failure: Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-process.yml b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-process.yml index 3853e66e366..a04da626e29 100644 --- a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-process.yml +++ b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-process.yml @@ -2,128 +2,155 @@ description: Pipeline for processing Process Event logs. processors: - set: + tag: set_event_category_dffe0229 field: event.category value: [process] - set: + tag: set_event_type_c5f986b8 field: event.type value: [start] if: ctx.sentinel_one_cloud_funnel?.event?.meta_event_name == 'PROCESSCREATION' - set: + tag: set_event_type_98a05a75 field: event.type value: [end] - if: ( - ctx.sentinel_one_cloud_funnel?.event?.type == 'ProcessExit' || ctx.sentinel_one_cloud_funnel?.event?.type == 'ProcessTermination' || - ctx.sentinel_one_cloud_funnel?.event?.type == 'Process Exit' || ctx.sentinel_one_cloud_funnel?.event?.type == 'Process Termination' - ) + if: ( ctx.sentinel_one_cloud_funnel?.event?.type == 'ProcessExit' || ctx.sentinel_one_cloud_funnel?.event?.type == 'ProcessTermination' || ctx.sentinel_one_cloud_funnel?.event?.type == 'Process Exit' || ctx.sentinel_one_cloud_funnel?.event?.type == 'Process Termination' ) - set: + tag: set_event_action_784fc211 field: event.action copy_from: event.type ignore_empty_value: true - rename: + tag: rename_json_k8sCluster_containerId_to_sentinel_one_cloud_funnel_event_k8s_cluster_container_id_174e75ab field: json.k8sCluster.containerId target_field: sentinel_one_cloud_funnel.event.k8s_cluster.container.id ignore_missing: true - set: + tag: set_container_id_a11673c8 field: container.id copy_from: sentinel_one_cloud_funnel.event.k8s_cluster.container.id ignore_empty_value: true - rename: + tag: rename_json_k8sCluster_containerImage_sha256_to_sentinel_one_cloud_funnel_event_k8s_cluster_container_image_sha256_997132df field: json.k8sCluster.containerImage.sha256 target_field: sentinel_one_cloud_funnel.event.k8s_cluster.container.image.sha256 ignore_missing: true - append: + tag: append_container_image_hash_all_7085001c field: container.image.hash.all value: '{{{sentinel_one_cloud_funnel.event.k8s_cluster.container.image.sha256}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.k8s_cluster?.container?.image?.sha256 != null - append: + tag: append_related_hash_fef1cce8 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.k8s_cluster.container.image.sha256}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.k8s_cluster?.container?.image?.sha256 != null - rename: + tag: rename_json_k8sCluster_containerImage_value_to_sentinel_one_cloud_funnel_event_k8s_cluster_container_image_value_4e464a3d field: json.k8sCluster.containerImage.value target_field: sentinel_one_cloud_funnel.event.k8s_cluster.container.image.value ignore_missing: true - set: + tag: set_container_image_name_6b959730 field: container.image.name copy_from: sentinel_one_cloud_funnel.event.k8s_cluster.container.image.value ignore_empty_value: true - rename: + tag: rename_json_k8sCluster_containerLabels_to_sentinel_one_cloud_funnel_event_k8s_cluster_container_labels_3938f7d7 field: json.k8sCluster.containerLabels target_field: sentinel_one_cloud_funnel.event.k8s_cluster.container.labels ignore_missing: true - set: + tag: set_container_labels_9d053fd0 field: container.labels copy_from: sentinel_one_cloud_funnel.event.k8s_cluster.container.labels ignore_empty_value: true - rename: + tag: rename_json_k8sCluster_containerName_to_sentinel_one_cloud_funnel_event_k8s_cluster_container_name_ee0d5ad3 field: json.k8sCluster.containerName target_field: sentinel_one_cloud_funnel.event.k8s_cluster.container.name ignore_missing: true - set: + tag: set_container_name_979f1054 field: container.name copy_from: sentinel_one_cloud_funnel.event.k8s_cluster.container.name ignore_empty_value: true - rename: + tag: rename_json_tgt_file_sha1_to_sentinel_one_cloud_funnel_event_tgt_file_sha1_4685387e field: json.tgt.file.sha1 target_field: sentinel_one_cloud_funnel.event.tgt.file.sha1 ignore_missing: true - set: + tag: set_file_hash_sha1_5c6ba116 field: file.hash.sha1 copy_from: sentinel_one_cloud_funnel.event.tgt.file.sha1 ignore_empty_value: true - append: + tag: append_related_hash_b56c7961 field: related.hash value: '{{{file.hash.sha1}}}' allow_duplicates: false if: ctx.file?.hash?.sha1 != null - rename: + tag: rename_json_tgt_process_user_to_sentinel_one_cloud_funnel_event_tgt_process_user_name_b291e32f field: json.tgt.process.user target_field: sentinel_one_cloud_funnel.event.tgt.process.user.name ignore_missing: true - append: + tag: append_related_user_72cd47f9 field: related.user value: '{{{sentinel_one_cloud_funnel.event.tgt.process.user.name}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.process?.user?.name != null - rename: + tag: rename_json_k8sCluster_controllerLabels_to_sentinel_one_cloud_funnel_event_k8s_cluster_controller_labels_10e1beab field: json.k8sCluster.controllerLabels target_field: sentinel_one_cloud_funnel.event.k8s_cluster.controller.labels ignore_missing: true - rename: + tag: rename_json_k8sCluster_controllerName_to_sentinel_one_cloud_funnel_event_k8s_cluster_controller_name_f2dcde6f field: json.k8sCluster.controllerName target_field: sentinel_one_cloud_funnel.event.k8s_cluster.controller.name ignore_missing: true - rename: + tag: rename_json_k8sCluster_controllerType_to_sentinel_one_cloud_funnel_event_k8s_cluster_controller_type_985eff4d field: json.k8sCluster.controllerType target_field: sentinel_one_cloud_funnel.event.k8s_cluster.controller.type ignore_missing: true - rename: + tag: rename_json_k8sCluster_name_to_sentinel_one_cloud_funnel_event_k8s_cluster_name_e94cd745 field: json.k8sCluster.name target_field: sentinel_one_cloud_funnel.event.k8s_cluster.name ignore_missing: true - rename: + tag: rename_json_k8sCluster_namespace_to_sentinel_one_cloud_funnel_event_k8s_cluster_namespace_value_cb9b4996 field: json.k8sCluster.namespace target_field: sentinel_one_cloud_funnel.event.k8s_cluster.namespace.value ignore_missing: true - rename: + tag: rename_json_k8sCluster_namespaceLabels_to_sentinel_one_cloud_funnel_event_k8s_cluster_namespace_labels_77ed3adf field: json.k8sCluster.namespaceLabels target_field: sentinel_one_cloud_funnel.event.k8s_cluster.namespace.labels ignore_missing: true - rename: + tag: rename_json_k8sCluster_nodeName_to_sentinel_one_cloud_funnel_event_k8s_cluster_node_name_494c1d46 field: json.k8sCluster.nodeName target_field: sentinel_one_cloud_funnel.event.k8s_cluster.node_name ignore_missing: true - rename: + tag: rename_json_k8sCluster_podLabels_to_sentinel_one_cloud_funnel_event_k8s_cluster_pod_labels_d3757fbf field: json.k8sCluster.podLabels target_field: sentinel_one_cloud_funnel.event.k8s_cluster.pod.labels ignore_missing: true - rename: + tag: rename_json_k8sCluster_podName_to_sentinel_one_cloud_funnel_event_k8s_cluster_pod_name_5bdc55a3 field: json.k8sCluster.podName target_field: sentinel_one_cloud_funnel.event.k8s_cluster.pod.name ignore_missing: true - rename: + tag: rename_json_src_process_reasonSignatureInvalid_to_sentinel_one_cloud_funnel_event_src_process_reason_signature_invalid_8220e978 field: json.src.process.reasonSignatureInvalid target_field: sentinel_one_cloud_funnel.event.src.process.reason_signature_invalid ignore_missing: true @@ -136,9 +163,11 @@ processors: if: ctx.json?.src?.process?.rpid != '' on_failure: - append: + tag: append_error_message_24bb4c90 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tgt_file_convictedBy_to_sentinel_one_cloud_funnel_event_tgt_file_convicted_by_9dae0cc9 field: json.tgt.file.convictedBy target_field: sentinel_one_cloud_funnel.event.tgt.file.convicted_by ignore_missing: true @@ -151,33 +180,41 @@ processors: if: ctx.json?.tgt?.process?.accessRights != '' on_failure: - append: + tag: append_error_message_658ceeba field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tgt_process_activeContent_hash_to_sentinel_one_cloud_funnel_event_tgt_process_active_content_hash_7e5be2bd field: json.tgt.process.activeContent.hash target_field: sentinel_one_cloud_funnel.event.tgt.process.active_content.hash ignore_missing: true - rename: + tag: rename_json_tgt_process_activeContent_id_to_sentinel_one_cloud_funnel_event_tgt_process_active_content_id_7796cb63 field: json.tgt.process.activeContent.id target_field: sentinel_one_cloud_funnel.event.tgt.process.active_content.id ignore_missing: true - rename: + tag: rename_json_tgt_process_activeContent_path_to_sentinel_one_cloud_funnel_event_tgt_process_active_content_path_c97e7e9f field: json.tgt.process.activeContent.path target_field: sentinel_one_cloud_funnel.event.tgt.process.active_content.path ignore_missing: true - rename: + tag: rename_json_tgt_process_activeContent_signedStatus_to_sentinel_one_cloud_funnel_event_tgt_process_active_content_signed_status_b1e15fcc field: json.tgt.process.activeContent.signedStatus target_field: sentinel_one_cloud_funnel.event.tgt.process.active_content.signed_status ignore_missing: true - rename: + tag: rename_json_tgt_process_activeContentType_to_sentinel_one_cloud_funnel_event_tgt_process_active_content_type_0e0027e7 field: json.tgt.process.activeContentType target_field: sentinel_one_cloud_funnel.event.tgt.process.active_content.type ignore_missing: true - rename: + tag: rename_json_tgt_process_cmdline_to_sentinel_one_cloud_funnel_event_tgt_process_cmd_line_812ccb5d field: json.tgt.process.cmdline target_field: sentinel_one_cloud_funnel.event.tgt.process.cmd_line ignore_missing: true - rename: + tag: rename_json_tgt_process_displayName_to_sentinel_one_cloud_funnel_event_tgt_process_display_name_abf63e69 field: json.tgt.process.displayName target_field: sentinel_one_cloud_funnel.event.tgt.process.display_name ignore_missing: true @@ -190,40 +227,49 @@ processors: if: ctx.json?.tgt?.process?.image?.binaryIsExecutable != '' on_failure: - append: + tag: append_error_message_16263fcf field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tgt_process_image_md5_to_sentinel_one_cloud_funnel_event_tgt_process_image_md5_edb32ef0 field: json.tgt.process.image.md5 target_field: sentinel_one_cloud_funnel.event.tgt.process.image.md5 ignore_missing: true - append: + tag: append_related_hash_e8d5a020 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.tgt.process.image.md5}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.process?.image?.md5 != null - rename: + tag: rename_json_tgt_process_image_path_to_sentinel_one_cloud_funnel_event_tgt_process_image_path_f507d268 field: json.tgt.process.image.path target_field: sentinel_one_cloud_funnel.event.tgt.process.image.path ignore_missing: true - rename: + tag: rename_json_tgt_process_image_sha1_to_sentinel_one_cloud_funnel_event_tgt_process_image_sha1_a2281558 field: json.tgt.process.image.sha1 target_field: sentinel_one_cloud_funnel.event.tgt.process.image.sha1 ignore_missing: true - append: + tag: append_related_hash_272c5210 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.tgt.process.image.sha1}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.process?.image?.sha1!= null - rename: + tag: rename_json_tgt_process_image_sha256_to_sentinel_one_cloud_funnel_event_tgt_process_image_sha256_a4a15668 field: json.tgt.process.image.sha256 target_field: sentinel_one_cloud_funnel.event.tgt.process.image.sha256 ignore_missing: true - append: + tag: append_related_hash_a2061640 field: related.hash value: '{{{sentinel_one_cloud_funnel.event.tgt.process.image.sha256}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.process?.image?.sha256 != null - rename: + tag: rename_json_tgt_process_integrityLevel_to_sentinel_one_cloud_funnel_event_tgt_process_integrity_level_1e2fbd25 field: json.tgt.process.integrityLevel target_field: sentinel_one_cloud_funnel.event.tgt.process.integrity_level ignore_missing: true @@ -236,6 +282,7 @@ processors: if: ctx.json?.tgt?.process?.isNative64Bit != '' on_failure: - append: + tag: append_error_message_7e1e78b8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -247,6 +294,7 @@ processors: if: ctx.json?.tgt?.process?.isRedirectCmdProcessor != '' on_failure: - append: + tag: append_error_message_5387e536 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -258,30 +306,37 @@ processors: if: ctx.json?.tgt?.process?.isStorylineRoot != '' on_failure: - append: + tag: append_error_message_bd000a95 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tgt_process_lUserName_to_sentinel_one_cloud_funnel_event_tgt_process_l_user_name_00afd15f field: json.tgt.process.lUserName target_field: sentinel_one_cloud_funnel.event.tgt.process.l_user.name ignore_missing: true - append: + tag: append_related_user_3d7736a5 field: related.user value: '{{{sentinel_one_cloud_funnel.event.tgt.process.l_user.name}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.process?.l_user?.name != null - rename: + tag: rename_json_tgt_process_name_to_sentinel_one_cloud_funnel_event_tgt_process_name_27916ab4 field: json.tgt.process.name target_field: sentinel_one_cloud_funnel.event.tgt.process.name ignore_missing: true - rename: + tag: rename_json_tgt_process_publisher_to_sentinel_one_cloud_funnel_event_tgt_process_publisher_c8fa2f52 field: json.tgt.process.publisher target_field: sentinel_one_cloud_funnel.event.tgt.process.publisher ignore_missing: true - set: + tag: set_process_code_signature_subject_name_17e74dae field: process.code_signature.subject_name copy_from: sentinel_one_cloud_funnel.event.tgt.process.publisher ignore_empty_value: true - rename: + tag: rename_json_tgt_process_reasonSignatureInvalid_to_sentinel_one_cloud_funnel_event_tgt_process_reason_signature_invalid_b95abc7c field: json.tgt.process.reasonSignatureInvalid target_field: sentinel_one_cloud_funnel.event.tgt.process.reason_signature_invalid ignore_missing: true @@ -294,17 +349,21 @@ processors: if: ctx.json?.tgt?.process?.sessionId != '' on_failure: - append: + tag: append_error_message_8f1c8019 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tgt_process_signedStatus_to_sentinel_one_cloud_funnel_event_tgt_process_signed_status_025c2f53 field: json.tgt.process.signedStatus target_field: sentinel_one_cloud_funnel.event.tgt.process.signed_status ignore_missing: true - set: + tag: set_process_code_signature_exists_ffb73e84 field: process.code_signature.exists value: true if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.process?.signed_status == 'signed' - set: + tag: set_process_code_signature_exists_4e52852f field: process.code_signature.exists value: false if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.process?.signed_status != 'signed' @@ -318,102 +377,127 @@ processors: if: ctx.json?.tgt?.process?.startTime != null && ctx.json.tgt.process.startTime != '' on_failure: - append: + tag: append_error_message_5fe39210 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tgt_process_storyline_id_to_sentinel_one_cloud_funnel_event_tgt_process_storyline_id_e1588021 field: json.tgt.process.storyline.id target_field: sentinel_one_cloud_funnel.event.tgt.process.storyline_id ignore_missing: true - rename: + tag: rename_json_tgt_process_subsystem_to_sentinel_one_cloud_funnel_event_tgt_process_subsystem_f2387370 field: json.tgt.process.subsystem target_field: sentinel_one_cloud_funnel.event.tgt.process.subsystem ignore_missing: true - rename: + tag: rename_json_tgt_process_uid_to_sentinel_one_cloud_funnel_event_tgt_process_uid_d0b00836 field: json.tgt.process.uid target_field: sentinel_one_cloud_funnel.event.tgt.process.uid ignore_missing: true - rename: + tag: rename_json_tgt_process_verifiedStatus_to_sentinel_one_cloud_funnel_event_tgt_process_verified_status_99559a2f field: json.tgt.process.verifiedStatus target_field: sentinel_one_cloud_funnel.event.tgt.process.verified_status ignore_missing: true - set: + tag: set_process_code_signature_trusted_213972e1 field: process.code_signature.trusted value: true if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.process?.verified_status == 'verified' - set: + tag: set_process_code_signature_trusted_79782acc field: process.code_signature.trusted value: false if: ctx.process?.code_signature?.exists == true && ctx.sentinel_one_cloud_funnel?.event?.tgt?.process?.verified_status != 'verified' - remove: + tag: remove_ba4ad84f field: - process.parent ignore_missing: true - rename: + tag: rename_process_to_process_parent_2c7cab59 field: process target_field: process.parent - append: + tag: append_process_user_name_3d861cac field: process.user.name value: '{{{sentinel_one_cloud_funnel.event.tgt.process.user.name}}}' allow_duplicates: false if: ctx.sentinel_one_cloud_funnel?.event?.tgt?.process?.user?.name != null - set: + tag: set_process_name_18431f12 field: process.name copy_from: sentinel_one_cloud_funnel.event.tgt.process.name ignore_empty_value: true - set: + tag: set_process_executable_23bea6ca field: process.executable copy_from: sentinel_one_cloud_funnel.event.tgt.process.image.path ignore_empty_value: true - convert: + tag: convert_sentinel_one_cloud_funnel_event_tgt_process_pid_to_process_pid_f22853e6 field: sentinel_one_cloud_funnel.event.tgt.process.pid target_field: process.pid type: long ignore_missing: true - set: + tag: set_process_thread_id_ff69c428 field: process.thread.id copy_from: sentinel_one_cloud_funnel.event.tgt.process.tid ignore_empty_value: true - set: + tag: set_process_entity_id_8f7b0b4b field: process.entity_id copy_from: sentinel_one_cloud_funnel.event.tgt.process.uid ignore_empty_value: true - set: + tag: set_process_command_line_3993e8e3 field: process.command_line copy_from: sentinel_one_cloud_funnel.event.tgt.process.cmd_line ignore_empty_value: true - set: + tag: set_process_hash_md5_62d3288f field: process.hash.md5 copy_from: sentinel_one_cloud_funnel.event.tgt.process.image.md5 ignore_empty_value: true - set: + tag: set_process_hash_sha1_910615c5 field: process.hash.sha1 copy_from: sentinel_one_cloud_funnel.event.tgt.process.image.sha1 ignore_empty_value: true - set: + tag: set_process_hash_sha256_ea8c63f5 field: process.hash.sha256 copy_from: sentinel_one_cloud_funnel.event.tgt.process.image.sha256 ignore_empty_value: true - set: + tag: set_process_real_user_id_3a4d0487 field: process.real_user.id copy_from: sentinel_one_cloud_funnel.event.tgt.process.r_user.uid ignore_empty_value: true - set: + tag: set_process_real_user_name_7609bb1a field: process.real_user.name copy_from: sentinel_one_cloud_funnel.event.tgt.process.r_user.name ignore_empty_value: true - set: + tag: set_process_start_20ffefb4 field: process.start copy_from: sentinel_one_cloud_funnel.event.tgt.process.start_time ignore_empty_value: true - set: + tag: set_process_title_af2d7746 field: process.title copy_from: sentinel_one_cloud_funnel.event.tgt.process.display_name ignore_empty_value: true - set: + tag: set_process_user_id_59e267df field: process.user.id copy_from: sentinel_one_cloud_funnel.event.tgt.process.e_user.uid ignore_empty_value: true - set: + tag: set_process_user_name_8ceb4c6e field: process.user.name copy_from: sentinel_one_cloud_funnel.event.tgt.process.user.name ignore_empty_value: true @@ -424,3 +508,11 @@ on_failure: Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-registry.yml b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-registry.yml index 7dc2bbe3334..19927ed052f 100644 --- a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-registry.yml +++ b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-registry.yml @@ -2,49 +2,51 @@ description: Pipeline for processing Registry Event logs. processors: - set: + tag: set_event_category_43fe0db5 field: event.category value: [registry] - set: + tag: set_event_type_2eec8b9a field: event.type value: [creation] if: ctx.sentinel_one_cloud_funnel?.event?.type != null && ctx.sentinel_one_cloud_funnel?.event?.type.toLowerCase().contains('create') - set: + tag: set_event_type_7e4897f1 field: event.type value: [deletion] if: ctx.sentinel_one_cloud_funnel?.event?.type != null && ctx.sentinel_one_cloud_funnel?.event?.type.toLowerCase().contains('delet') - set: + tag: set_event_type_ef142c4d field: event.type value: [change] if: ctx.sentinel_one_cloud_funnel?.event?.type != null && (ctx.sentinel_one_cloud_funnel?.event?.type.toLowerCase().contains('change') || ctx.sentinel_one_cloud_funnel?.event?.type.toLowerCase().contains('modif')) - set: + tag: set_event_action_95f90b81 field: event.action value: [creation] - if: ctx.sentinel_one_cloud_funnel?.event?.type != null && - ( - ctx.sentinel_one_cloud_funnel?.event?.meta_event_name.toLowerCase().contains('regvaluecreate') || - ctx.sentinel_one_cloud_funnel?.event?.meta_event_name.toLowerCase().contains('regkeycreate') - ) + if: ctx.sentinel_one_cloud_funnel?.event?.type != null && ( ctx.sentinel_one_cloud_funnel?.event?.meta_event_name.toLowerCase().contains('regvaluecreate') || ctx.sentinel_one_cloud_funnel?.event?.meta_event_name.toLowerCase().contains('regkeycreate') ) - set: + tag: set_event_action_5609fac7 field: event.action value: [modification] if: ctx.sentinel_one_cloud_funnel?.event?.type != null && ctx.sentinel_one_cloud_funnel?.event?.meta_event_name.toLowerCase().contains('regvaluemodified') - set: + tag: set_event_action_6c79fe12 field: event.action value: [deletion] - if: ctx.sentinel_one_cloud_funnel?.event?.type != null && - ( - ctx.sentinel_one_cloud_funnel?.event?.meta_event_name.toLowerCase().contains('regvaluedelete') || - ctx.sentinel_one_cloud_funnel?.event?.meta_event_name.toLowerCase().contains('regkeydelete') - ) + if: ctx.sentinel_one_cloud_funnel?.event?.type != null && ( ctx.sentinel_one_cloud_funnel?.event?.meta_event_name.toLowerCase().contains('regvaluedelete') || ctx.sentinel_one_cloud_funnel?.event?.meta_event_name.toLowerCase().contains('regkeydelete') ) - rename: + tag: rename_json_registry_keyPath_to_sentinel_one_cloud_funnel_event_registry_key_path_143e00d4 field: json.registry.keyPath target_field: sentinel_one_cloud_funnel.event.registry.key.path ignore_missing: true - set: + tag: set_registry_path_c320dc78 field: registry.path copy_from: sentinel_one_cloud_funnel.event.registry.key.path ignore_empty_value: true - script: + tag: script_4ee2dd33 lang: painless if: ctx.registry?.path instanceof String && ctx.registry.path != "" source: |- @@ -54,26 +56,32 @@ processors: ctx.registry.value = ctx.registry.path.substring(idx+1); } - rename: + tag: rename_json_registry_value_to_sentinel_one_cloud_funnel_event_registry_val_a63421c6 field: json.registry.value target_field: sentinel_one_cloud_funnel.event.registry.val ignore_missing: true - set: + tag: set_registry_data_strings_75eee7cc field: registry.data.strings copy_from: sentinel_one_cloud_funnel.event.registry.val ignore_empty_value: true - rename: + tag: rename_json_registry_valueType_to_sentinel_one_cloud_funnel_event_registry_value_type_9815fbc6 field: json.registry.valueType target_field: sentinel_one_cloud_funnel.event.registry.value.type ignore_missing: true - set: + tag: set_registry_data_type_fe6e4ae0 field: registry.data.type copy_from: sentinel_one_cloud_funnel.event.registry.value.type ignore_empty_value: true - rename: + tag: rename_json_registry_keyUid_to_sentinel_one_cloud_funnel_event_registry_key_uid_f0cff858 field: json.registry.keyUid target_field: sentinel_one_cloud_funnel.event.registry.key.uid ignore_missing: true - rename: + tag: rename_json_registry_oldValue_to_sentinel_one_cloud_funnel_event_registry_old_value_detail_c2d817f8 field: json.registry.oldValue target_field: sentinel_one_cloud_funnel.event.registry.old_value.detail ignore_missing: true @@ -86,6 +94,7 @@ processors: if: ctx.json?.registry?.oldValueFullSize != '' on_failure: - append: + tag: append_error_message_d5adbeb1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -97,9 +106,11 @@ processors: if: ctx.json?.registry?.oldValueIsComplete != '' on_failure: - append: + tag: append_error_message_bcb5b4af field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_registry_oldValueType_to_sentinel_one_cloud_funnel_event_registry_old_value_type_0df81887 field: json.registry.oldValueType target_field: sentinel_one_cloud_funnel.event.registry.old_value.type ignore_missing: true @@ -112,6 +123,7 @@ processors: if: ctx.json?.registry?.valueFullSize != '' on_failure: - append: + tag: append_error_message_48d701f2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -123,9 +135,11 @@ processors: if: ctx.json?.registry?.valueIsComplete != '' on_failure: - append: + tag: append_error_message_7de8a374 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_registry_valueType_to_sentinel_one_cloud_funnel_event_registry_value_type_9815fbc6 field: json.registry.valueType target_field: sentinel_one_cloud_funnel.event.registry.value.type ignore_missing: true @@ -136,3 +150,11 @@ on_failure: Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-scheduled-task.yml b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-scheduled-task.yml index 26aca18d648..5e19a431cf0 100644 --- a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-scheduled-task.yml +++ b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-scheduled-task.yml @@ -11,55 +11,68 @@ processors: if: ctx.json?.tgt?.file?.creationTime != null && ctx.json.tgt.file.creationTime != '' on_failure: - append: + tag: append_error_message_3a00d0dc field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_file_created_d1ec61e1 field: file.created copy_from: sentinel_one_cloud_funnel.event.tgt.file.creation_time ignore_empty_value: true - rename: + tag: rename_json_tgt_file_extension_to_sentinel_one_cloud_funnel_event_tgt_file_extension_0c7969fc field: json.tgt.file.extension target_field: sentinel_one_cloud_funnel.event.tgt.file.extension ignore_missing: true - set: + tag: set_file_extension_e5c5c6d4 field: file.extension copy_from: sentinel_one_cloud_funnel.event.tgt.file.extension ignore_empty_value: true - rename: + tag: rename_json_tgt_file_md5_to_sentinel_one_cloud_funnel_event_tgt_file_md5_f59548a0 field: json.tgt.file.md5 target_field: sentinel_one_cloud_funnel.event.tgt.file.md5 ignore_missing: true - set: + tag: set_file_hash_md5_b5abcf60 field: file.hash.md5 copy_from: sentinel_one_cloud_funnel.event.tgt.file.md5 ignore_empty_value: true - append: + tag: append_related_hash_6af42da3 field: related.hash value: '{{{file.hash.md5}}}' allow_duplicates: false if: ctx.file?.hash?.md5 != null - rename: + tag: rename_json_tgt_file_sha1_to_sentinel_one_cloud_funnel_event_tgt_file_sha1_4685387e field: json.tgt.file.sha1 target_field: sentinel_one_cloud_funnel.event.tgt.file.sha1 ignore_missing: true - set: + tag: set_file_hash_sha1_5c6ba116 field: file.hash.sha1 copy_from: sentinel_one_cloud_funnel.event.tgt.file.sha1 ignore_empty_value: true - append: + tag: append_related_hash_b56c7961 field: related.hash value: '{{{file.hash.sha1}}}' allow_duplicates: false if: ctx.file?.hash?.sha1 != null - rename: + tag: rename_json_tgt_file_sha256_to_sentinel_one_cloud_funnel_event_tgt_file_sha256_94584cca field: json.tgt.file.sha256 target_field: sentinel_one_cloud_funnel.event.tgt.file.sha256 ignore_missing: true - set: + tag: set_file_hash_sha256_df14789e field: file.hash.sha256 copy_from: sentinel_one_cloud_funnel.event.tgt.file.sha256 ignore_empty_value: true - append: + tag: append_related_hash_017aef51 field: related.hash value: '{{{file.hash.sha256}}}' allow_duplicates: false @@ -74,17 +87,21 @@ processors: if: ctx.json?.tgt?.file?.modificationTime != null && ctx.json.tgt.file.modificationTime != '' on_failure: - append: + tag: append_error_message_d1a23c4b field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_file_mtime_88ff3fe4 field: file.mtime copy_from: sentinel_one_cloud_funnel.event.tgt.file.modification_time ignore_empty_value: true - rename: + tag: rename_json_tgt_file_path_to_sentinel_one_cloud_funnel_event_tgt_file_path_39609c16 field: json.tgt.file.path target_field: sentinel_one_cloud_funnel.event.tgt.file.path ignore_missing: true - set: + tag: set_file_path_527bee9a field: file.path copy_from: sentinel_one_cloud_funnel.event.tgt.file.path ignore_empty_value: true @@ -97,37 +114,46 @@ processors: if: ctx.json?.tgt?.file?.size != '' on_failure: - append: + tag: append_error_message_96037225 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: + tag: set_file_size_53ce742a field: file.size copy_from: sentinel_one_cloud_funnel.event.tgt.file.size ignore_empty_value: true - rename: + tag: rename_json_tgt_file_type_to_sentinel_one_cloud_funnel_event_tgt_file_type_d8855d28 field: json.tgt.file.type target_field: sentinel_one_cloud_funnel.event.tgt.file.type ignore_missing: true - set: + tag: set_file_type_6b11f406 field: file.type copy_from: sentinel_one_cloud_funnel.event.tgt.file.type ignore_empty_value: true - rename: + tag: rename_json_task_name_to_sentinel_one_cloud_funnel_event_task_name_89d63b66 field: json.task.name target_field: sentinel_one_cloud_funnel.event.task.name ignore_missing: true - rename: + tag: rename_json_task_path_to_sentinel_one_cloud_funnel_event_task_path_a1b7920a field: json.task.path target_field: sentinel_one_cloud_funnel.event.task.path ignore_missing: true - rename: + tag: rename_json_tgt_file_description_to_sentinel_one_cloud_funnel_event_tgt_file_description_54771acc field: json.tgt.file.description target_field: sentinel_one_cloud_funnel.event.tgt.file.description ignore_missing: true - rename: + tag: rename_json_tgt_file_id_to_sentinel_one_cloud_funnel_event_tgt_file_id_76a97c5e field: json.tgt.file.id target_field: sentinel_one_cloud_funnel.event.tgt.file.id ignore_missing: true - rename: + tag: rename_json_tgt_file_internalName_to_sentinel_one_cloud_funnel_event_tgt_file_internal_name_84e0ac81 field: json.tgt.file.internalName target_field: sentinel_one_cloud_funnel.event.tgt.file.internal_name ignore_missing: true @@ -140,9 +166,11 @@ processors: if: ctx.json?.tgt?.file?.isExecutable != '' on_failure: - append: + tag: append_error_message_7406da9a field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tgt_file_location_to_sentinel_one_cloud_funnel_event_tgt_file_location_90526cb6 field: json.tgt.file.location target_field: sentinel_one_cloud_funnel.event.tgt.file.location ignore_missing: true @@ -153,3 +181,11 @@ on_failure: Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-threat-intelligence-indicator.yml b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-threat-intelligence-indicator.yml index 30b2f6427aa..7f6ce00a60c 100644 --- a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-threat-intelligence-indicator.yml +++ b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-threat-intelligence-indicator.yml @@ -2,16 +2,20 @@ description: Pipeline for processing Threat Intelligence Indicator Event logs. processors: - set: + tag: set_event_category_cceb7c22 field: event.category value: [threat] - set: + tag: set_event_type_97bb78b9 field: event.type value: [indicator] - rename: + tag: rename_json_tiIndicator_addedBy_to_sentinel_one_cloud_funnel_event_ti_indicator_added_by_8bf7b266 field: json.tiIndicator.addedBy target_field: sentinel_one_cloud_funnel.event.ti_indicator.added_by ignore_missing: true - rename: + tag: rename_json_tiIndicator_comparisonMethod_to_sentinel_one_cloud_funnel_event_ti_indicator_comparison_method_1a0daf46 field: json.tiIndicator.comparisonMethod target_field: sentinel_one_cloud_funnel.event.ti_indicator.comparison_method ignore_missing: true @@ -25,13 +29,16 @@ processors: if: ctx.json?.tiIndicator?.modificationTime != null && ctx.json.tiIndicator.modificationTime != '' on_failure: - append: + tag: append_error_message_40edfd55 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tiIndicator_name_to_sentinel_one_cloud_funnel_event_ti_indicator_name_d840c5cb field: json.tiIndicator.name target_field: sentinel_one_cloud_funnel.event.ti_indicator.name ignore_missing: true - rename: + tag: rename_json_tiindicator_originalEvent_id_to_sentinel_one_cloud_funnel_event_ti_indicator_original_event_id_3d84f7e6 field: json.tiindicator.originalEvent.id target_field: sentinel_one_cloud_funnel.event.ti_indicator.original_event.id ignore_missing: true @@ -44,6 +51,7 @@ processors: if: ctx.json?.tiindicator?.originalEvent?.index != '' on_failure: - append: + tag: append_error_message_66844991 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -56,21 +64,26 @@ processors: if: ctx.json?.tiindicator?.originalEvent?.time != null && ctx.json.tiindicator.originalEvent.time != '' on_failure: - append: + tag: append_error_message_c9cbd0d4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tiindicator_originalEvent_traceId_to_sentinel_one_cloud_funnel_event_ti_indicator_original_event_trace_id_a639061f field: json.tiindicator.originalEvent.traceId target_field: sentinel_one_cloud_funnel.event.ti_indicator.original_event.trace_id ignore_missing: true - rename: + tag: rename_json_tiIndicator_source_to_sentinel_one_cloud_funnel_event_ti_indicator_source_452c36f3 field: json.tiIndicator.source target_field: sentinel_one_cloud_funnel.event.ti_indicator.source ignore_missing: true - rename: + tag: rename_json_tiIndicator_type_to_sentinel_one_cloud_funnel_event_ti_indicator_type_b30049a1 field: json.tiIndicator.type target_field: sentinel_one_cloud_funnel.event.ti_indicator.type ignore_missing: true - rename: + tag: rename_json_tiIndicator_uid_to_sentinel_one_cloud_funnel_event_ti_indicator_uid_95b027bd field: json.tiIndicator.uid target_field: sentinel_one_cloud_funnel.event.ti_indicator.uid ignore_missing: true @@ -84,6 +97,7 @@ processors: if: ctx.json?.tiIndicator?.uploadTime != null && ctx.json.tiIndicator.uploadTime != '' on_failure: - append: + tag: append_error_message_aa16ff68 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -96,9 +110,11 @@ processors: if: ctx.json?.tiIndicator?.validUntil != null && ctx.json.tiIndicator.validUntil != '' on_failure: - append: + tag: append_error_message_d1637106 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_tiIndicator_value_to_sentinel_one_cloud_funnel_event_ti_indicator_value_3b461759 field: json.tiIndicator.value target_field: sentinel_one_cloud_funnel.event.ti_indicator.value ignore_missing: true @@ -109,3 +125,11 @@ on_failure: Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-url.yml b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-url.yml index ef45aefb98c..a613c9bb488 100644 --- a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-url.yml +++ b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/pipeline-url.yml @@ -2,26 +2,32 @@ description: Pipeline for processing Url Event logs. processors: - rename: + tag: rename_json_url_address_to_sentinel_one_cloud_funnel_event_url_address_ab881b30 field: json.url.address target_field: sentinel_one_cloud_funnel.event.url.address ignore_missing: true - set: + tag: set_url_original_3e91113e field: url.original copy_from: sentinel_one_cloud_funnel.event.url.address ignore_empty_value: true - uri_parts: + tag: uri_parts_url_original_c57c6a60 field: url.original keep_original: true ignore_missing: true - set: + tag: set_destination_address_0d605210 field: destination.address copy_from: url.domain ignore_empty_value: true - registered_domain: + tag: registered_domain_destination_address_to_destination_ae93f7ee field: destination.address target_field: destination if: ctx.destination?.address != null - rename: + tag: rename_json_event_url_action_to_sentinel_one_cloud_funnel_event_url_action_f74cb416 field: json.event.url.action target_field: sentinel_one_cloud_funnel.event.url.action ignore_missing: true @@ -32,3 +38,11 @@ on_failure: Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + tag: set_pipeline_error_to_event_kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/sentinel_one_cloud_funnel/manifest.yml b/packages/sentinel_one_cloud_funnel/manifest.yml index 389daa797ae..29bad52debe 100644 --- a/packages/sentinel_one_cloud_funnel/manifest.yml +++ b/packages/sentinel_one_cloud_funnel/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: sentinel_one_cloud_funnel title: SentinelOne Cloud Funnel -version: "1.13.1" +version: "1.13.2" description: Collect logs from SentinelOne Cloud Funnel with Elastic Agent. type: integration categories: From 36080f163a29f10357449cfd52257306d5f3e00e Mon Sep 17 00:00:00 2001 From: kcreddy Date: Thu, 6 Aug 2026 19:44:49 +0530 Subject: [PATCH 2/2] ssi: update changelog links to PR #20573 Update the link field in each new changelog entry from the tracking issue to the merged pull request. Updates #20558 --- packages/entityanalytics_okta/changelog.yml | 2 +- packages/eset_protect/changelog.yml | 2 +- packages/first_epss/changelog.yml | 2 +- packages/forgerock/changelog.yml | 2 +- packages/gcp/changelog.yml | 2 +- packages/github/changelog.yml | 2 +- packages/gitlab/changelog.yml | 2 +- packages/google_scc/changelog.yml | 2 +- packages/google_secops/changelog.yml | 2 +- packages/google_workspace/changelog.yml | 2 +- packages/greenhouse/changelog.yml | 2 +- packages/ibm_qradar/changelog.yml | 2 +- packages/imperva_cloud_waf/changelog.yml | 2 +- packages/infoblox_bloxone_ddi/changelog.yml | 2 +- packages/island_browser/changelog.yml | 2 +- packages/jamf_compliance_reporter/changelog.yml | 2 +- packages/jamf_pro/changelog.yml | 2 +- packages/jamf_protect/changelog.yml | 2 +- packages/jumpcloud/changelog.yml | 2 +- packages/keeper_security_siem_integration/changelog.yml | 2 +- packages/keycloak/changelog.yml | 2 +- packages/lastpass/changelog.yml | 2 +- packages/lumos/changelog.yml | 2 +- packages/lyve_cloud/changelog.yml | 2 +- packages/m365_defender/changelog.yml | 2 +- packages/mattermost/changelog.yml | 2 +- packages/menlo/changelog.yml | 2 +- packages/microsoft_defender_cloud/changelog.yml | 2 +- packages/microsoft_defender_endpoint/changelog.yml | 2 +- packages/microsoft_exchange_online_message_trace/changelog.yml | 2 +- packages/microsoft_intune/changelog.yml | 2 +- packages/microsoft_sentinel/changelog.yml | 2 +- packages/mimecast/changelog.yml | 2 +- packages/miniflux/changelog.yml | 2 +- packages/netskope/changelog.yml | 2 +- packages/okta/changelog.yml | 2 +- packages/opencanary/changelog.yml | 2 +- packages/panw_cortex_xdr/changelog.yml | 2 +- packages/ping_federate/changelog.yml | 2 +- packages/ping_one/changelog.yml | 2 +- packages/pps/changelog.yml | 2 +- packages/prisma_cloud/changelog.yml | 2 +- packages/proofpoint_essentials/changelog.yml | 2 +- packages/proofpoint_itm/changelog.yml | 2 +- packages/proofpoint_on_demand/changelog.yml | 2 +- packages/proofpoint_tap/changelog.yml | 2 +- packages/qualys_vmdr/changelog.yml | 2 +- packages/rapid7_insightvm/changelog.yml | 2 +- packages/santa/changelog.yml | 2 +- packages/sentinel_one/changelog.yml | 2 +- packages/sentinel_one_cloud_funnel/changelog.yml | 2 +- 51 files changed, 51 insertions(+), 51 deletions(-) diff --git a/packages/entityanalytics_okta/changelog.yml b/packages/entityanalytics_okta/changelog.yml index 48e76c8814d..7762230c277 100644 --- a/packages/entityanalytics_okta/changelog.yml +++ b/packages/entityanalytics_okta/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "3.4.0" changes: - description: Add the Advanced Analytics (UEBA) subcategory diff --git a/packages/eset_protect/changelog.yml b/packages/eset_protect/changelog.yml index dd926d42ec1..0c11239ff8a 100644 --- a/packages/eset_protect/changelog.yml +++ b/packages/eset_protect/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "2.5.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/first_epss/changelog.yml b/packages/first_epss/changelog.yml index acbb9fd288f..0bc59b52b79 100644 --- a/packages/first_epss/changelog.yml +++ b/packages/first_epss/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.4.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/forgerock/changelog.yml b/packages/forgerock/changelog.yml index 178187ddc39..abc99f93dfc 100644 --- a/packages/forgerock/changelog.yml +++ b/packages/forgerock/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.24.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/gcp/changelog.yml b/packages/gcp/changelog.yml index bd6130a7ea7..658787563ba 100644 --- a/packages/gcp/changelog.yml +++ b/packages/gcp/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "2.50.2" changes: - description: Add security category to the Load Balancing and VPC Flow data streams to match their policy template categories. diff --git a/packages/github/changelog.yml b/packages/github/changelog.yml index 8e46f8ef914..1114476b381 100644 --- a/packages/github/changelog.yml +++ b/packages/github/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "2.25.2" changes: - description: Add `event.action` and `event.reason` to the `issues` data stream. diff --git a/packages/gitlab/changelog.yml b/packages/gitlab/changelog.yml index 6774a9b3ecd..493edd3b033 100644 --- a/packages/gitlab/changelog.yml +++ b/packages/gitlab/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "2.6.1" changes: - description: Normalize `custom_message` field to handle both string and object values. diff --git a/packages/google_scc/changelog.yml b/packages/google_scc/changelog.yml index 960b8702730..8d091de0f30 100644 --- a/packages/google_scc/changelog.yml +++ b/packages/google_scc/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "2.6.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/google_secops/changelog.yml b/packages/google_secops/changelog.yml index 4d400cb6596..3f95638ef4a 100644 --- a/packages/google_secops/changelog.yml +++ b/packages/google_secops/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.3.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/google_workspace/changelog.yml b/packages/google_workspace/changelog.yml index 335ff8a5f27..563bb4d478f 100644 --- a/packages/google_workspace/changelog.yml +++ b/packages/google_workspace/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "3.5.2" changes: - description: Add missing ECS event categorization for the gcp, rules, calendar, group_enterprise, context_aware_access, meet, chat, chrome, login, device, keep, and vault data streams. diff --git a/packages/greenhouse/changelog.yml b/packages/greenhouse/changelog.yml index f5af6dc3bba..bc805829e87 100644 --- a/packages/greenhouse/changelog.yml +++ b/packages/greenhouse/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "0.1.0" changes: - description: Initial release. diff --git a/packages/ibm_qradar/changelog.yml b/packages/ibm_qradar/changelog.yml index 0d5643a5899..e52e8293b1f 100644 --- a/packages/ibm_qradar/changelog.yml +++ b/packages/ibm_qradar/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.1.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/imperva_cloud_waf/changelog.yml b/packages/imperva_cloud_waf/changelog.yml index 1334d9c460c..80689de70bf 100644 --- a/packages/imperva_cloud_waf/changelog.yml +++ b/packages/imperva_cloud_waf/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.15.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/infoblox_bloxone_ddi/changelog.yml b/packages/infoblox_bloxone_ddi/changelog.yml index f2e4e8bc791..2cd379f1ac8 100644 --- a/packages/infoblox_bloxone_ddi/changelog.yml +++ b/packages/infoblox_bloxone_ddi/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.23.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/island_browser/changelog.yml b/packages/island_browser/changelog.yml index f3a5d59e493..2bf64953b79 100644 --- a/packages/island_browser/changelog.yml +++ b/packages/island_browser/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.1.3" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/jamf_compliance_reporter/changelog.yml b/packages/jamf_compliance_reporter/changelog.yml index 41d5368edd3..33796b2f62d 100644 --- a/packages/jamf_compliance_reporter/changelog.yml +++ b/packages/jamf_compliance_reporter/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.16.0" changes: - description: Update Kibana constraint to support 9.0.0. diff --git a/packages/jamf_pro/changelog.yml b/packages/jamf_pro/changelog.yml index 879cb1dcf5b..2bf72d9cba0 100644 --- a/packages/jamf_pro/changelog.yml +++ b/packages/jamf_pro/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "2.0.0" changes: - description: Add latest transform to the inventory data stream. diff --git a/packages/jamf_protect/changelog.yml b/packages/jamf_protect/changelog.yml index be9396ff7d4..fadb6d9331c 100644 --- a/packages/jamf_protect/changelog.yml +++ b/packages/jamf_protect/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "3.4.0" changes: - description: Set 'elastic' owner type. diff --git a/packages/jumpcloud/changelog.yml b/packages/jumpcloud/changelog.yml index e1c419de1fd..76a96d6ce0f 100644 --- a/packages/jumpcloud/changelog.yml +++ b/packages/jumpcloud/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.22.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/keeper_security_siem_integration/changelog.yml b/packages/keeper_security_siem_integration/changelog.yml index 7a699c2f741..4f8603e1c92 100644 --- a/packages/keeper_security_siem_integration/changelog.yml +++ b/packages/keeper_security_siem_integration/changelog.yml @@ -2,7 +2,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "0.1.0" changes: - description: Initial release of Keeper Security agentless integration diff --git a/packages/keycloak/changelog.yml b/packages/keycloak/changelog.yml index 78c710510a8..efbea415a5a 100644 --- a/packages/keycloak/changelog.yml +++ b/packages/keycloak/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.32.0" changes: - description: Set 'elastic' owner type. diff --git a/packages/lastpass/changelog.yml b/packages/lastpass/changelog.yml index daafb62902c..79c1b0575ae 100644 --- a/packages/lastpass/changelog.yml +++ b/packages/lastpass/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.23.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/lumos/changelog.yml b/packages/lumos/changelog.yml index 9af54d9dbc1..ca9258b7482 100644 --- a/packages/lumos/changelog.yml +++ b/packages/lumos/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.8.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/lyve_cloud/changelog.yml b/packages/lyve_cloud/changelog.yml index b7cac37cb3a..ce97211d55b 100644 --- a/packages/lyve_cloud/changelog.yml +++ b/packages/lyve_cloud/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.17.1" changes: - description: Add `region` variable in the aws-s3 input to fix compatibility with Elastic Agent 8.19.12+ when using non-AWS S3-compatible buckets. diff --git a/packages/m365_defender/changelog.yml b/packages/m365_defender/changelog.yml index fea997691e3..036595ecc3c 100644 --- a/packages/m365_defender/changelog.yml +++ b/packages/m365_defender/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "5.15.1" changes: - description: Fix `alert` and `incident` pagination to advance the `$filter` time boundary (`lastUpdateDateTime gt `) with `$skip` held at 0, instead of incrementing `$skip`. This avoids the Graph security API `$skip <= 500` limit (which caused `400` errors and stalled catch-up under high volume) and drains a backlog in a single interval. Also correct the `alert` `batch_size` default from 2000 to 1000, matching the API `$top` cap. diff --git a/packages/mattermost/changelog.yml b/packages/mattermost/changelog.yml index 2a324f85d44..b99669d8847 100644 --- a/packages/mattermost/changelog.yml +++ b/packages/mattermost/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "2.5.0" changes: - description: Add dashboard. diff --git a/packages/menlo/changelog.yml b/packages/menlo/changelog.yml index fb5b4d9145d..944ee233eac 100644 --- a/packages/menlo/changelog.yml +++ b/packages/menlo/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.8.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/microsoft_defender_cloud/changelog.yml b/packages/microsoft_defender_cloud/changelog.yml index eb7eac6559b..a19e44fc9f5 100644 --- a/packages/microsoft_defender_cloud/changelog.yml +++ b/packages/microsoft_defender_cloud/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "3.5.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/microsoft_defender_endpoint/changelog.yml b/packages/microsoft_defender_endpoint/changelog.yml index 24e8dda0ae3..4ea45de089f 100644 --- a/packages/microsoft_defender_endpoint/changelog.yml +++ b/packages/microsoft_defender_endpoint/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "4.9.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/microsoft_exchange_online_message_trace/changelog.yml b/packages/microsoft_exchange_online_message_trace/changelog.yml index 8955e7fe350..c05cf3712f6 100644 --- a/packages/microsoft_exchange_online_message_trace/changelog.yml +++ b/packages/microsoft_exchange_online_message_trace/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "2.2.5" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/microsoft_intune/changelog.yml b/packages/microsoft_intune/changelog.yml index 8000cc7a196..aebf9f109c0 100644 --- a/packages/microsoft_intune/changelog.yml +++ b/packages/microsoft_intune/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: 0.1.0 changes: - description: Initial release of package. diff --git a/packages/microsoft_sentinel/changelog.yml b/packages/microsoft_sentinel/changelog.yml index 015e835088e..a0570f1dc88 100644 --- a/packages/microsoft_sentinel/changelog.yml +++ b/packages/microsoft_sentinel/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.4.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/mimecast/changelog.yml b/packages/mimecast/changelog.yml index 5d35dd9e48a..819e6f5ef3f 100644 --- a/packages/mimecast/changelog.yml +++ b/packages/mimecast/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "3.5.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/miniflux/changelog.yml b/packages/miniflux/changelog.yml index e864d96e77e..58e270b0c7d 100644 --- a/packages/miniflux/changelog.yml +++ b/packages/miniflux/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.2.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/netskope/changelog.yml b/packages/netskope/changelog.yml index b520df4f3c0..b7f3143cbc8 100644 --- a/packages/netskope/changelog.yml +++ b/packages/netskope/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "3.1.4" changes: - description: Add content_type and override_content_type options to the transaction data stream for the azure-blob-storage input. diff --git a/packages/okta/changelog.yml b/packages/okta/changelog.yml index 33cdd7f5127..67d805f7c2e 100644 --- a/packages/okta/changelog.yml +++ b/packages/okta/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "3.15.0" changes: - description: Use new `release` field for agentless deployment mode to establish as beta. diff --git a/packages/opencanary/changelog.yml b/packages/opencanary/changelog.yml index 4404f01315f..5075cde4aa9 100644 --- a/packages/opencanary/changelog.yml +++ b/packages/opencanary/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.0.0" changes: - description: Release package as GA. diff --git a/packages/panw_cortex_xdr/changelog.yml b/packages/panw_cortex_xdr/changelog.yml index b6670655c34..46cbca87a2c 100644 --- a/packages/panw_cortex_xdr/changelog.yml +++ b/packages/panw_cortex_xdr/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "2.7.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/ping_federate/changelog.yml b/packages/ping_federate/changelog.yml index 6dd41121fe0..5eedd1f63e3 100644 --- a/packages/ping_federate/changelog.yml +++ b/packages/ping_federate/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.2.0" changes: - description: Support IPv6 source addresses with brackets in audit data stream. diff --git a/packages/ping_one/changelog.yml b/packages/ping_one/changelog.yml index abc3d12d36b..6b108d9c2c9 100644 --- a/packages/ping_one/changelog.yml +++ b/packages/ping_one/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.24.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/pps/changelog.yml b/packages/pps/changelog.yml index 9081a21d34d..94a5f5d70f2 100644 --- a/packages/pps/changelog.yml +++ b/packages/pps/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.2.1" changes: - description: Fix parsing for plain usernames and space-padded dates for syslog messages from Pleasant Password Server. diff --git a/packages/prisma_cloud/changelog.yml b/packages/prisma_cloud/changelog.yml index 66f323a507f..c0a8775a3c9 100644 --- a/packages/prisma_cloud/changelog.yml +++ b/packages/prisma_cloud/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "4.2.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/proofpoint_essentials/changelog.yml b/packages/proofpoint_essentials/changelog.yml index cb718ac0511..55e387e3222 100644 --- a/packages/proofpoint_essentials/changelog.yml +++ b/packages/proofpoint_essentials/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.1.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/proofpoint_itm/changelog.yml b/packages/proofpoint_itm/changelog.yml index a8a51247956..b84e76dc6a2 100644 --- a/packages/proofpoint_itm/changelog.yml +++ b/packages/proofpoint_itm/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.1.1" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/proofpoint_on_demand/changelog.yml b/packages/proofpoint_on_demand/changelog.yml index 536e5ce93c6..899ffc76d33 100644 --- a/packages/proofpoint_on_demand/changelog.yml +++ b/packages/proofpoint_on_demand/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.10.2" changes: - description: Fix kibana.version constraint to use tilde ranges for intermediate version bounds. diff --git a/packages/proofpoint_tap/changelog.yml b/packages/proofpoint_tap/changelog.yml index 02b66916ca3..e8475ef753b 100644 --- a/packages/proofpoint_tap/changelog.yml +++ b/packages/proofpoint_tap/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.31.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/qualys_vmdr/changelog.yml b/packages/qualys_vmdr/changelog.yml index 27154264c8a..1238732c97c 100644 --- a/packages/qualys_vmdr/changelog.yml +++ b/packages/qualys_vmdr/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "6.19.4" changes: - description: Fix deduplication in the asset_host_detection data stream. diff --git a/packages/rapid7_insightvm/changelog.yml b/packages/rapid7_insightvm/changelog.yml index d6683884d6b..cc1b8592631 100644 --- a/packages/rapid7_insightvm/changelog.yml +++ b/packages/rapid7_insightvm/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "2.9.2" changes: - description: Add missing field definition for rapid7.insightvm.asset.same.reintroduced. diff --git a/packages/santa/changelog.yml b/packages/santa/changelog.yml index 92065bd810b..a8b227fd274 100644 --- a/packages/santa/changelog.yml +++ b/packages/santa/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "3.24.0" changes: - description: Update Kibana constraint to support 9.0.0. diff --git a/packages/sentinel_one/changelog.yml b/packages/sentinel_one/changelog.yml index e13d65b20cb..550ce3167c5 100644 --- a/packages/sentinel_one/changelog.yml +++ b/packages/sentinel_one/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "2.11.3" changes: - description: Fix error handling for first phase of agent data collection. diff --git a/packages/sentinel_one_cloud_funnel/changelog.yml b/packages/sentinel_one_cloud_funnel/changelog.yml index f1525580954..b35dad9f474 100644 --- a/packages/sentinel_one_cloud_funnel/changelog.yml +++ b/packages/sentinel_one_cloud_funnel/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers. type: enhancement - link: https://github.com/elastic/integrations/issues/20558 + link: https://github.com/elastic/integrations/pull/20573 - version: "1.13.1" changes: - description: Remove duplicated installation instructions from the documentation