diff --git a/packages/ti_opencti/changelog.yml b/packages/ti_opencti/changelog.yml index 7ae0c620a61..b2a554177b3 100644 --- a/packages/ti_opencti/changelog.yml +++ b/packages/ti_opencti/changelog.yml @@ -1,4 +1,15 @@ # newer versions go on top +- version: "2.15.4" + changes: + - description: Fix handling of service DLL observables to avoid invalid and incorrect renames, and simplify. + type: bugfix + link: https://github.com/elastic/integrations/pull/20569 + - description: Fix invalid `?.` syntax in Mustache templates. + type: bugfix + link: https://github.com/elastic/integrations/pull/20569 + - description: Copy `threat.indicator.file.size` value without unnecessary stringification. + type: bugfix + link: https://github.com/elastic/integrations/pull/20569 - version: "2.15.3" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json b/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json new file mode 100644 index 00000000000..cecc50ec440 --- /dev/null +++ b/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json @@ -0,0 +1,86 @@ +{ + "events": [ + { + "id": "abc123ab-c123-abc1-23ab-c123abc123ab", + "standard_id": "indicator--123abc12-3abc-123a-bc12-3abc123abc12", + "is_inferred": false, + "revoked": true, + "confidence": 30, + "lang": "en", + "created": "2023-01-17T06:44:58.868Z", + "modified": "2023-01-17T09:29:44.973Z", + "pattern_type": "stix", + "pattern_version": "2.1", + "pattern": "[process:command_line = './gedit-bin --new-window']", + "name": "Example process ./gedit-bin --new-window", + "description": null, + "valid_from": "2017-08-23T14:00:51.000Z", + "valid_until": "2017-10-22T14:00:51.000Z", + "x_opencti_score": 50, + "x_opencti_detection": false, + "x_opencti_main_observable_type": "Process", + "createdBy": { + "identity_class": "tester", + "name": "Manual" + }, + "objectMarking": [], + "objectLabel": [], + "killChainPhases": [], + "externalReferences": { + "edges": [] + }, + "observables": { + "edges": [ + { + "node": { + "id": "def123de-123d-f123-ef12-def123def123", + "standard_id": "process--a12b3a12-3a12-3a12-3a12-3a12b3a12b3a", + "entity_type": "Process", + "observable_value": "1221", + "pid": 1221, + "command_line": "./gedit-bin --new-window", + "serviceDlls": { + "edges": [ + { + "node": { + "hashes": [ + { + "algorithm": "MD5", + "hash": "6ad06d0d468f76fdc23e561054eab063" + }, + { + "algorithm": "SHA-256", + "hash": "841514e050d5c7d0e9b431de1c1379bcbcc0d0bb290290b1cfbd57e0f21fcc36" + } + ], + "size": 44544, + "name": "evil-service.dll", + "name_enc": null, + "magic_number_hex": null, + "mime_type": "application/vnd.microsoft.portable-executable", + "ctime": "2023-01-10T00:00:00.000Z", + "mtime": "2023-01-11T00:00:00.000Z", + "atime": null, + "x_opencti_additional_names": [ + "innocent-name.dll" + ], + "obsContent": { + "payload_bin": null, + "url": "https://malware.example.test/evil-service.dll", + "encryption_algorithm": null, + "decryption_key": null + } + } + } + ] + } + } + } + ], + "pageInfo": { + "globalCount": 1 + } + } + } + ] +} diff --git a/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json-expected.json b/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json-expected.json new file mode 100644 index 00000000000..489aebf210c --- /dev/null +++ b/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json-expected.json @@ -0,0 +1,124 @@ +{ + "expected": [ + { + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "threat" + ], + "created": "2023-01-17T06:44:58.868Z", + "id": "abc123ab-c123-abc1-23ab-c123abc123ab", + "kind": "enrichment", + "type": [ + "indicator" + ] + }, + "opencti": { + "indicator": { + "creator_identity_class": "tester", + "detection": false, + "invalid_or_revoked_from": "2017-10-22T14:00:51.000Z", + "is_inferred": false, + "lang": "en", + "observables_count": 1, + "pattern": "[process:command_line = './gedit-bin --new-window']", + "pattern_type": "stix", + "pattern_version": "2.1", + "revoked": true, + "score": 50, + "standard_id": "indicator--123abc12-3abc-123a-bc12-3abc123abc12", + "valid_from": "2017-08-23T14:00:51.000Z", + "valid_until": "2017-10-22T14:00:51.000Z" + }, + "observable": { + "process": { + "command_line": "./gedit-bin --new-window", + "entity_type": "Process", + "id": "def123de-123d-f123-ef12-def123def123", + "pid": 1221, + "service_dll": [ + { + "additional_names": [ + "innocent-name.dll" + ], + "content": { + "url": "https://malware.example.test/evil-service.dll" + }, + "ctime": "2023-01-10T00:00:00.000Z", + "hash": { + "md5": "6ad06d0d468f76fdc23e561054eab063", + "sha256": "841514e050d5c7d0e9b431de1c1379bcbcc0d0bb290290b1cfbd57e0f21fcc36" + }, + "mime_type": "application/vnd.microsoft.portable-executable", + "mtime": "2023-01-11T00:00:00.000Z", + "name": "evil-service.dll", + "size": 44544 + } + ], + "standard_id": "process--a12b3a12-3a12-3a12-3a12-3a12b3a12b3a", + "value": "1221" + } + } + }, + "related": { + "hash": [ + "841514e050d5c7d0e9b431de1c1379bcbcc0d0bb290290b1cfbd57e0f21fcc36", + "6ad06d0d468f76fdc23e561054eab063" + ], + "hosts": [ + "malware.example.test" + ] + }, + "tags": [ + "forwarded", + "opencti-indicator", + "ecs-indicator-detail" + ], + "threat": { + "feed": { + "dashboard_id": "ti_opencti-83b2bef0-591c-11ee-ba5f-49a63bb985cd", + "description": "Indicator data from OpenCTI", + "name": "OpenCTI", + "reference": "https://docs.opencti.io/latest/usage/overview/" + }, + "indicator": { + "confidence": "Medium", + "file": { + "created": "2023-01-10T00:00:00.000Z", + "extension": [ + "dll", + "dll" + ], + "hash": { + "md5": "6ad06d0d468f76fdc23e561054eab063", + "sha256": "841514e050d5c7d0e9b431de1c1379bcbcc0d0bb290290b1cfbd57e0f21fcc36" + }, + "mime_type": "application/vnd.microsoft.portable-executable", + "mtime": "2023-01-11T00:00:00.000Z", + "name": [ + "evil-service.dll", + "innocent-name.dll" + ], + "size": 44544, + "type": "file" + }, + "modified_at": "2023-01-17T09:29:44.973Z", + "name": "Example process ./gedit-bin --new-window", + "provider": "Manual", + "reference": "https://demo.opencti.io/dashboard/observations/indicators/abc123ab-c123-abc1-23ab-c123abc123ab", + "type": "process", + "url": { + "domain": "malware.example.test", + "extension": "dll", + "full": "https://malware.example.test/evil-service.dll", + "original": "https://malware.example.test/evil-service.dll", + "path": "/evil-service.dll", + "scheme": "https" + } + } + } + } + ] +} diff --git a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml index bc6b8726481..35736769194 100644 --- a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml @@ -285,50 +285,32 @@ processors: target_field: _ingest._value.node.content ignore_missing: true - - foreach: - field: observables.edges - ignore_missing: true - processor: - foreach: - field: _ingest._value.node.serviceDlls.edges - ignore_missing: true - processor: - rename: - field: _ingest._value.node.x_opencti_additional_names - target_field: _ingest.node._value.additional_names - - - foreach: - field: observables.edges - ignore_missing: true - processor: - foreach: - field: _ingest._value.node.serviceDlls.edges - ignore_missing: true - processor: - rename: - field: _ingest._value.node.obsContent - target_field: _ingest._value.node.content - - - foreach: - field: observables.edges - ignore_missing: true - processor: - foreach: - field: _ingest._value.node.serviceDlls.edges - ignore_missing: true - processor: - rename: - field: _ingest._value.node - target_field: _ingest._value - - - foreach: - field: observables.edges - ignore_missing: true - processor: - rename: - field: _ingest._value.node.serviceDlls.edges - target_field: _ingest._value.node.service_dlls - ignore_missing: true + - script: + description: Unwrap service DLL nodes from the GraphQL connection into service_dll + lang: painless + source: | + if (ctx.observables?.edges instanceof List) { + for (def edge : ctx.observables.edges) { + def dllEdges = edge.node?.serviceDlls?.edges; + if (dllEdges instanceof List) { + def dlls = []; + for (def dllEdge : dllEdges) { + def dll = dllEdge.node; + if (dll != null) { + if (dll.containsKey('x_opencti_additional_names')) { + dll.additional_names = dll.remove('x_opencti_additional_names'); + } + if (dll.containsKey('obsContent')) { + dll.content = dll.remove('obsContent'); + } + dlls.add(dll); + } + } + edge.node.service_dll = dlls; + } + edge.node?.remove('serviceDlls'); + } + } - script: description: Merge array of objects into a single object for startup_info @@ -371,9 +353,9 @@ processors: } observable.remove('hashes'); } - if (observable.containsKey('service_dlls')) { - for (int ii = 0; ii < observable.service_dlls.length; ii++) { - Map serviceDll = observable.service_dlls[ii]; + if (observable.containsKey('service_dll')) { + for (int ii = 0; ii < observable.service_dll.length; ii++) { + Map serviceDll = observable.service_dll[ii]; if (serviceDll.containsKey('hashes')) { def ecsHash = hashesToECS(serviceDll.hashes); if (ecsHash.size() > 0) { @@ -561,7 +543,7 @@ processors: processor: append: field: _tmp_found_urls - value: "{{{_ingest._value.content?.url}}}" + value: "{{{_ingest._value.content.url}}}" allow_duplicates: false - foreach: field: opencti.observable.media_content @@ -581,7 +563,7 @@ processors: processor: append: field: _tmp_found_urls - value: "{{{_ingest._value.content?.url}}}" + value: "{{{_ingest._value.content.url}}}" allow_duplicates: false - script: description: Remove null or empty items diff --git a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_file.yml b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_file.yml index 39b1f8f5c2e..a8e68cf7ad5 100644 --- a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_file.yml +++ b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_file.yml @@ -77,7 +77,7 @@ processors: - set: field: _tmp_file.size - value: "{{{_ingest._value.size}}}" + copy_from: _ingest._value.size ignore_empty_value: true - set: diff --git a/packages/ti_opencti/manifest.yml b/packages/ti_opencti/manifest.yml index 4d082f785db..1eb5f43c2cd 100644 --- a/packages/ti_opencti/manifest.yml +++ b/packages/ti_opencti/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.4.0" name: ti_opencti title: OpenCTI -version: "2.15.3" +version: "2.15.4" description: "Ingest threat intelligence indicators from OpenCTI with Elastic Agent." type: integration source: