From e8768ca5e211e7bb571974bc26771cd4843e86c9 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 6 Aug 2026 09:48:28 +0200 Subject: [PATCH 1/3] [ti_opencti] Fix service DLL handling and other minor pipeline issues The original service DLL handling had two broken renames: - To `_ingest.node._value.additional_names` rather than `_ingest._value....` (from the foreach processor). - From `_ingest._value.node` to `_ingest._value`, which failed because the target exists. It also produced `service_dlls`, while the mappings and downstream processors expect `service_dll`. It's fixed and simplified by replacing the four nested foreach processors with a single Painless script that unwraps each `serviceDlls.edges[].node` into `service_dll` and renames the OpenCTI-specific fields at the same time. Invalid `?.` syntax is removed from Mustache templates (it's for Painless, not Mustache). The `threat.indicator.file.size` value is set with `copy_from` rather than a template to avoid unnecessary stringification. --- packages/ti_opencti/changelog.yml | 11 ++ .../test-process-with-service-dll.json | 86 ++++++++++++ ...rocess-with-service-dll.json-expected.json | 124 ++++++++++++++++++ .../elasticsearch/ingest_pipeline/default.yml | 80 +++++------ .../ingest_pipeline/ecs_from_file.yml | 2 +- packages/ti_opencti/manifest.yml | 2 +- 6 files changed, 254 insertions(+), 51 deletions(-) create mode 100644 packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json create mode 100644 packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json-expected.json diff --git a/packages/ti_opencti/changelog.yml b/packages/ti_opencti/changelog.yml index 7ae0c620a61..3d1d8af2521 100644 --- a/packages/ti_opencti/changelog.yml +++ b/packages/ti_opencti/changelog.yml @@ -1,4 +1,15 @@ # newer versions go on top +- version: "2.15.4" + changes: + - description: Fix handling of service DLL observables to avoid invalid and incorrect renames, and simplify. + type: bugfix + link: https://github.com/elastic/integrations/pull/1 + - description: Fix invalid `?.` syntax in a Mustache templates. + type: bugfix + link: https://github.com/elastic/integrations/pull/1 + - description: Copy `threat.indicator.file.size` value without unnecessary stringification. + type: bugfix + link: https://github.com/elastic/integrations/pull/1 - version: "2.15.3" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json b/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json new file mode 100644 index 00000000000..cecc50ec440 --- /dev/null +++ b/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json @@ -0,0 +1,86 @@ +{ + "events": [ + { + "id": "abc123ab-c123-abc1-23ab-c123abc123ab", + "standard_id": "indicator--123abc12-3abc-123a-bc12-3abc123abc12", + "is_inferred": false, + "revoked": true, + "confidence": 30, + "lang": "en", + "created": "2023-01-17T06:44:58.868Z", + "modified": "2023-01-17T09:29:44.973Z", + "pattern_type": "stix", + "pattern_version": "2.1", + "pattern": "[process:command_line = './gedit-bin --new-window']", + "name": "Example process ./gedit-bin --new-window", + "description": null, + "valid_from": "2017-08-23T14:00:51.000Z", + "valid_until": "2017-10-22T14:00:51.000Z", + "x_opencti_score": 50, + "x_opencti_detection": false, + "x_opencti_main_observable_type": "Process", + "createdBy": { + "identity_class": "tester", + "name": "Manual" + }, + "objectMarking": [], + "objectLabel": [], + "killChainPhases": [], + "externalReferences": { + "edges": [] + }, + "observables": { + "edges": [ + { + "node": { + "id": "def123de-123d-f123-ef12-def123def123", + "standard_id": "process--a12b3a12-3a12-3a12-3a12-3a12b3a12b3a", + "entity_type": "Process", + "observable_value": "1221", + "pid": 1221, + "command_line": "./gedit-bin --new-window", + "serviceDlls": { + "edges": [ + { + "node": { + "hashes": [ + { + "algorithm": "MD5", + "hash": "6ad06d0d468f76fdc23e561054eab063" + }, + { + "algorithm": "SHA-256", + "hash": "841514e050d5c7d0e9b431de1c1379bcbcc0d0bb290290b1cfbd57e0f21fcc36" + } + ], + "size": 44544, + "name": "evil-service.dll", + "name_enc": null, + "magic_number_hex": null, + "mime_type": "application/vnd.microsoft.portable-executable", + "ctime": "2023-01-10T00:00:00.000Z", + "mtime": "2023-01-11T00:00:00.000Z", + "atime": null, + "x_opencti_additional_names": [ + "innocent-name.dll" + ], + "obsContent": { + "payload_bin": null, + "url": "https://malware.example.test/evil-service.dll", + "encryption_algorithm": null, + "decryption_key": null + } + } + } + ] + } + } + } + ], + "pageInfo": { + "globalCount": 1 + } + } + } + ] +} diff --git a/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json-expected.json b/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json-expected.json new file mode 100644 index 00000000000..489aebf210c --- /dev/null +++ b/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json-expected.json @@ -0,0 +1,124 @@ +{ + "expected": [ + { + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "threat" + ], + "created": "2023-01-17T06:44:58.868Z", + "id": "abc123ab-c123-abc1-23ab-c123abc123ab", + "kind": "enrichment", + "type": [ + "indicator" + ] + }, + "opencti": { + "indicator": { + "creator_identity_class": "tester", + "detection": false, + "invalid_or_revoked_from": "2017-10-22T14:00:51.000Z", + "is_inferred": false, + "lang": "en", + "observables_count": 1, + "pattern": "[process:command_line = './gedit-bin --new-window']", + "pattern_type": "stix", + "pattern_version": "2.1", + "revoked": true, + "score": 50, + "standard_id": "indicator--123abc12-3abc-123a-bc12-3abc123abc12", + "valid_from": "2017-08-23T14:00:51.000Z", + "valid_until": "2017-10-22T14:00:51.000Z" + }, + "observable": { + "process": { + "command_line": "./gedit-bin --new-window", + "entity_type": "Process", + "id": "def123de-123d-f123-ef12-def123def123", + "pid": 1221, + "service_dll": [ + { + "additional_names": [ + "innocent-name.dll" + ], + "content": { + "url": "https://malware.example.test/evil-service.dll" + }, + "ctime": "2023-01-10T00:00:00.000Z", + "hash": { + "md5": "6ad06d0d468f76fdc23e561054eab063", + "sha256": "841514e050d5c7d0e9b431de1c1379bcbcc0d0bb290290b1cfbd57e0f21fcc36" + }, + "mime_type": "application/vnd.microsoft.portable-executable", + "mtime": "2023-01-11T00:00:00.000Z", + "name": "evil-service.dll", + "size": 44544 + } + ], + "standard_id": "process--a12b3a12-3a12-3a12-3a12-3a12b3a12b3a", + "value": "1221" + } + } + }, + "related": { + "hash": [ + "841514e050d5c7d0e9b431de1c1379bcbcc0d0bb290290b1cfbd57e0f21fcc36", + "6ad06d0d468f76fdc23e561054eab063" + ], + "hosts": [ + "malware.example.test" + ] + }, + "tags": [ + "forwarded", + "opencti-indicator", + "ecs-indicator-detail" + ], + "threat": { + "feed": { + "dashboard_id": "ti_opencti-83b2bef0-591c-11ee-ba5f-49a63bb985cd", + "description": "Indicator data from OpenCTI", + "name": "OpenCTI", + "reference": "https://docs.opencti.io/latest/usage/overview/" + }, + "indicator": { + "confidence": "Medium", + "file": { + "created": "2023-01-10T00:00:00.000Z", + "extension": [ + "dll", + "dll" + ], + "hash": { + "md5": "6ad06d0d468f76fdc23e561054eab063", + "sha256": "841514e050d5c7d0e9b431de1c1379bcbcc0d0bb290290b1cfbd57e0f21fcc36" + }, + "mime_type": "application/vnd.microsoft.portable-executable", + "mtime": "2023-01-11T00:00:00.000Z", + "name": [ + "evil-service.dll", + "innocent-name.dll" + ], + "size": 44544, + "type": "file" + }, + "modified_at": "2023-01-17T09:29:44.973Z", + "name": "Example process ./gedit-bin --new-window", + "provider": "Manual", + "reference": "https://demo.opencti.io/dashboard/observations/indicators/abc123ab-c123-abc1-23ab-c123abc123ab", + "type": "process", + "url": { + "domain": "malware.example.test", + "extension": "dll", + "full": "https://malware.example.test/evil-service.dll", + "original": "https://malware.example.test/evil-service.dll", + "path": "/evil-service.dll", + "scheme": "https" + } + } + } + } + ] +} diff --git a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml index bc6b8726481..35736769194 100644 --- a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml @@ -285,50 +285,32 @@ processors: target_field: _ingest._value.node.content ignore_missing: true - - foreach: - field: observables.edges - ignore_missing: true - processor: - foreach: - field: _ingest._value.node.serviceDlls.edges - ignore_missing: true - processor: - rename: - field: _ingest._value.node.x_opencti_additional_names - target_field: _ingest.node._value.additional_names - - - foreach: - field: observables.edges - ignore_missing: true - processor: - foreach: - field: _ingest._value.node.serviceDlls.edges - ignore_missing: true - processor: - rename: - field: _ingest._value.node.obsContent - target_field: _ingest._value.node.content - - - foreach: - field: observables.edges - ignore_missing: true - processor: - foreach: - field: _ingest._value.node.serviceDlls.edges - ignore_missing: true - processor: - rename: - field: _ingest._value.node - target_field: _ingest._value - - - foreach: - field: observables.edges - ignore_missing: true - processor: - rename: - field: _ingest._value.node.serviceDlls.edges - target_field: _ingest._value.node.service_dlls - ignore_missing: true + - script: + description: Unwrap service DLL nodes from the GraphQL connection into service_dll + lang: painless + source: | + if (ctx.observables?.edges instanceof List) { + for (def edge : ctx.observables.edges) { + def dllEdges = edge.node?.serviceDlls?.edges; + if (dllEdges instanceof List) { + def dlls = []; + for (def dllEdge : dllEdges) { + def dll = dllEdge.node; + if (dll != null) { + if (dll.containsKey('x_opencti_additional_names')) { + dll.additional_names = dll.remove('x_opencti_additional_names'); + } + if (dll.containsKey('obsContent')) { + dll.content = dll.remove('obsContent'); + } + dlls.add(dll); + } + } + edge.node.service_dll = dlls; + } + edge.node?.remove('serviceDlls'); + } + } - script: description: Merge array of objects into a single object for startup_info @@ -371,9 +353,9 @@ processors: } observable.remove('hashes'); } - if (observable.containsKey('service_dlls')) { - for (int ii = 0; ii < observable.service_dlls.length; ii++) { - Map serviceDll = observable.service_dlls[ii]; + if (observable.containsKey('service_dll')) { + for (int ii = 0; ii < observable.service_dll.length; ii++) { + Map serviceDll = observable.service_dll[ii]; if (serviceDll.containsKey('hashes')) { def ecsHash = hashesToECS(serviceDll.hashes); if (ecsHash.size() > 0) { @@ -561,7 +543,7 @@ processors: processor: append: field: _tmp_found_urls - value: "{{{_ingest._value.content?.url}}}" + value: "{{{_ingest._value.content.url}}}" allow_duplicates: false - foreach: field: opencti.observable.media_content @@ -581,7 +563,7 @@ processors: processor: append: field: _tmp_found_urls - value: "{{{_ingest._value.content?.url}}}" + value: "{{{_ingest._value.content.url}}}" allow_duplicates: false - script: description: Remove null or empty items diff --git a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_file.yml b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_file.yml index 39b1f8f5c2e..a8e68cf7ad5 100644 --- a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_file.yml +++ b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_file.yml @@ -77,7 +77,7 @@ processors: - set: field: _tmp_file.size - value: "{{{_ingest._value.size}}}" + copy_from: _ingest._value.size ignore_empty_value: true - set: diff --git a/packages/ti_opencti/manifest.yml b/packages/ti_opencti/manifest.yml index 4d082f785db..1eb5f43c2cd 100644 --- a/packages/ti_opencti/manifest.yml +++ b/packages/ti_opencti/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.4.0" name: ti_opencti title: OpenCTI -version: "2.15.3" +version: "2.15.4" description: "Ingest threat intelligence indicators from OpenCTI with Elastic Agent." type: integration source: From 97d4c3f142449e8de2cadd4f1786c87fcb4ba03f Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 6 Aug 2026 11:00:09 +0200 Subject: [PATCH 2/3] Update links in changelog entry. --- packages/ti_opencti/changelog.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/ti_opencti/changelog.yml b/packages/ti_opencti/changelog.yml index 3d1d8af2521..311b1b19417 100644 --- a/packages/ti_opencti/changelog.yml +++ b/packages/ti_opencti/changelog.yml @@ -3,13 +3,13 @@ changes: - description: Fix handling of service DLL observables to avoid invalid and incorrect renames, and simplify. type: bugfix - link: https://github.com/elastic/integrations/pull/1 + link: https://github.com/elastic/integrations/pull/20569 - description: Fix invalid `?.` syntax in a Mustache templates. type: bugfix - link: https://github.com/elastic/integrations/pull/1 + link: https://github.com/elastic/integrations/pull/20569 - description: Copy `threat.indicator.file.size` value without unnecessary stringification. type: bugfix - link: https://github.com/elastic/integrations/pull/1 + link: https://github.com/elastic/integrations/pull/20569 - version: "2.15.3" changes: - description: Set agentless deployment mode `release` field to `ga`. From 1d00510462c286b91a41ace85282d8162ddfb786 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 6 Aug 2026 11:01:22 +0200 Subject: [PATCH 3/3] Fix typo in changelog entry. --- packages/ti_opencti/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ti_opencti/changelog.yml b/packages/ti_opencti/changelog.yml index 311b1b19417..b2a554177b3 100644 --- a/packages/ti_opencti/changelog.yml +++ b/packages/ti_opencti/changelog.yml @@ -4,7 +4,7 @@ - description: Fix handling of service DLL observables to avoid invalid and incorrect renames, and simplify. type: bugfix link: https://github.com/elastic/integrations/pull/20569 - - description: Fix invalid `?.` syntax in a Mustache templates. + - description: Fix invalid `?.` syntax in Mustache templates. type: bugfix link: https://github.com/elastic/integrations/pull/20569 - description: Copy `threat.indicator.file.size` value without unnecessary stringification.