From 1def9ce7f05f2d9bdb0abbafff48d5cf2039d5cc Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Wed, 5 Aug 2026 20:08:31 +0200 Subject: [PATCH 1/5] [Azure] Add AMQP-over-WebSockets support to Event Hub integrations --- .../_dev/build/docs/README.md | 8 +++- packages/azure_ai_foundry/changelog.yml | 5 +++ .../logs/agent/stream/azure-eventhub.yml.hbs | 3 ++ .../data_stream/logs/manifest.yml | 15 +++++++ packages/azure_ai_foundry/docs/README.md | 7 +++- packages/azure_ai_foundry/manifest.yml | 2 +- .../_dev/build/docs/README.md | 6 +++ packages/azure_app_service/changelog.yml | 5 +++ .../agent/stream/azure-eventhub.yml.hbs | 3 ++ packages/azure_app_service/docs/README.md | 6 +++ packages/azure_app_service/manifest.yml | 17 +++++++- .../azure_functions/_dev/build/docs/README.md | 10 +++++ packages/azure_functions/changelog.yml | 5 +++ .../agent/stream/azure-eventhub.yml.hbs | 3 ++ .../data_stream/functionapplogs/manifest.yml | 15 +++++++ packages/azure_functions/docs/README.md | 10 +++++ packages/azure_functions/manifest.yml | 2 +- packages/azure_logs/_dev/build/docs/README.md | 39 +++++++++++++++---- packages/azure_logs/agent/input/input.yml.hbs | 5 ++- packages/azure_logs/changelog.yml | 5 +++ packages/azure_logs/docs/README.md | 39 +++++++++++++++---- packages/azure_logs/manifest.yml | 17 +++++++- .../azure_openai/_dev/build/docs/README.md | 8 +++- packages/azure_openai/changelog.yml | 5 +++ .../logs/agent/stream/azure-eventhub.yml.hbs | 3 ++ .../data_stream/logs/manifest.yml | 15 +++++++ packages/azure_openai/docs/README.md | 7 +++- packages/azure_openai/manifest.yml | 2 +- 28 files changed, 243 insertions(+), 24 deletions(-) diff --git a/packages/azure_ai_foundry/_dev/build/docs/README.md b/packages/azure_ai_foundry/_dev/build/docs/README.md index ef38d29b970..3927357c457 100644 --- a/packages/azure_ai_foundry/_dev/build/docs/README.md +++ b/packages/azure_ai_foundry/_dev/build/docs/README.md @@ -36,6 +36,12 @@ Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for m **Authentication (Event Hub):** The Event Hub input supports two authentication methods: **connection string** (default) and **client secret** (Microsoft Entra ID). For setup steps, required RBAC roles (Azure Event Hubs Data Receiver, Storage Blob Data Contributor), and configuration options, see the [Azure Logs integration](https://docs.elastic.co/integrations/azure) or [Filebeat azure-eventhub input](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-eventhub.html) documentation. +#### AMQP-over-WebSockets and proxy support + +By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. + +This option requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. + #### Native logging The Microsoft Foundry provides native logging and monitoring to track the telemetry of the service. The `Audit` and `RequestResponse` log categories come under the native logging. However, the default logging doesn't log the inputs and outputs of the service. This is useful to ensure that the services operates as expected. @@ -115,4 +121,4 @@ For more details on ECS fields, check the [ECS Field Reference](https://www.elas {{fields "metrics"}} ## Alerting Rule Template -{{alertRuleTemplates}} \ No newline at end of file +{{alertRuleTemplates}} diff --git a/packages/azure_ai_foundry/changelog.yml b/packages/azure_ai_foundry/changelog.yml index 076a050347b..e2e495c8043 100644 --- a/packages/azure_ai_foundry/changelog.yml +++ b/packages/azure_ai_foundry/changelog.yml @@ -1,3 +1,8 @@ +- version: "0.13.0" + changes: + - description: Add support for AMQP-over-WebSockets transport protocol and proxy configuration. + type: enhancement + link: https://github.com/elastic/obs-integration-team/issues/973 - version: "0.12.0" changes: - description: Publish Agentless to Elastic Managed integrations name change. diff --git a/packages/azure_ai_foundry/data_stream/logs/agent/stream/azure-eventhub.yml.hbs b/packages/azure_ai_foundry/data_stream/logs/agent/stream/azure-eventhub.yml.hbs index 6b8c5be15d1..4b0a538ac48 100644 --- a/packages/azure_ai_foundry/data_stream/logs/agent/stream/azure-eventhub.yml.hbs +++ b/packages/azure_ai_foundry/data_stream/logs/agent/stream/azure-eventhub.yml.hbs @@ -43,6 +43,9 @@ storage_account_key: {{storage_account_key}} {{#if resource_manager_endpoint}} resource_manager_endpoint: {{resource_manager_endpoint}} {{/if}} +{{#if transport}} +transport: {{transport}} +{{/if}} tags: {{#if preserve_original_event}} - preserve_original_event diff --git a/packages/azure_ai_foundry/data_stream/logs/manifest.yml b/packages/azure_ai_foundry/data_stream/logs/manifest.yml index 1977114118e..3a54804c267 100644 --- a/packages/azure_ai_foundry/data_stream/logs/manifest.yml +++ b/packages/azure_ai_foundry/data_stream/logs/manifest.yml @@ -113,6 +113,21 @@ streams: description: >- (Optional when **Authentication Type** is **Client Secret**) Microsoft Entra ID authority endpoint. Defaults to https://login.microsoftonline.com (Azure Public Cloud). Change for other Azure environments: Azure Government (https://login.microsoftonline.us), Azure China (https://login.chinacloudapi.cn), or Azure Germany (https://login.microsoftonline.de). + - name: transport + type: select + title: Event Hubs transport protocol + multi: false + required: true + show_user: false + default: amqp + options: + - text: AMQP + value: amqp + - text: AMQP-over-WebSockets + value: websocket + description: > + (Processor v2 only) The transport protocol used to connect to Event Hubs. + Supported values are AMQP and AMQP-over-WebSockets. Defaults to AMQP. - name: preserve_original_event required: true show_user: true diff --git a/packages/azure_ai_foundry/docs/README.md b/packages/azure_ai_foundry/docs/README.md index 394a1017add..af644c91746 100644 --- a/packages/azure_ai_foundry/docs/README.md +++ b/packages/azure_ai_foundry/docs/README.md @@ -36,6 +36,12 @@ Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for m **Authentication (Event Hub):** The Event Hub input supports two authentication methods: **connection string** (default) and **client secret** (Microsoft Entra ID). For setup steps, required RBAC roles (Azure Event Hubs Data Receiver, Storage Blob Data Contributor), and configuration options, see the [Azure Logs integration](https://docs.elastic.co/integrations/azure) or [Filebeat azure-eventhub input](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-eventhub.html) documentation. +#### AMQP-over-WebSockets and proxy support + +By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. + +This option requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. + #### Native logging The Microsoft Foundry provides native logging and monitoring to track the telemetry of the service. The `Audit` and `RequestResponse` log categories come under the native logging. However, the default logging doesn't log the inputs and outputs of the service. This is useful to ensure that the services operates as expected. @@ -503,4 +509,3 @@ The following alert rule templates are available: **[Microsoft Foundry] Provisioned Utilization above threshold** - diff --git a/packages/azure_ai_foundry/manifest.yml b/packages/azure_ai_foundry/manifest.yml index 9e5c45820e3..d1414bac8c2 100644 --- a/packages/azure_ai_foundry/manifest.yml +++ b/packages/azure_ai_foundry/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.4.0 name: azure_ai_foundry title: "Microsoft Foundry" -version: "0.12.0" +version: "0.13.0" source: license: "Elastic-2.0" description: "Collects Microsoft Foundry logs and metrics" diff --git a/packages/azure_app_service/_dev/build/docs/README.md b/packages/azure_app_service/_dev/build/docs/README.md index 7ecc790ae2f..9ccf44e717c 100644 --- a/packages/azure_app_service/_dev/build/docs/README.md +++ b/packages/azure_app_service/_dev/build/docs/README.md @@ -23,6 +23,12 @@ Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for m **Authentication (Event Hub):** The Event Hub input supports two authentication methods: **connection string** (default) and **client secret** (Microsoft Entra ID). For setup steps, required RBAC roles (Azure Event Hubs Data Receiver, Storage Blob Data Contributor), and configuration options, see the [Azure Logs integration](https://docs.elastic.co/integrations/azure) or [Filebeat azure-eventhub input](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-eventhub.html) documentation. +### AMQP-over-WebSockets and proxy support + +By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. + +This option requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. + ## App Service Logs Collects different types of logs from Azure App Service via Event Hub. diff --git a/packages/azure_app_service/changelog.yml b/packages/azure_app_service/changelog.yml index 7068c5dcf57..df89e4f875c 100644 --- a/packages/azure_app_service/changelog.yml +++ b/packages/azure_app_service/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.2.0" + changes: + - description: Add support for AMQP-over-WebSockets transport protocol and proxy configuration. + type: enhancement + link: https://github.com/elastic/obs-integration-team/issues/973 - version: "1.1.0" changes: - description: Add client secret authentication support for Azure Event Hub with RBAC. diff --git a/packages/azure_app_service/data_stream/app_service_logs/agent/stream/azure-eventhub.yml.hbs b/packages/azure_app_service/data_stream/app_service_logs/agent/stream/azure-eventhub.yml.hbs index 80366f47eec..4e8518c49ed 100644 --- a/packages/azure_app_service/data_stream/app_service_logs/agent/stream/azure-eventhub.yml.hbs +++ b/packages/azure_app_service/data_stream/app_service_logs/agent/stream/azure-eventhub.yml.hbs @@ -43,6 +43,9 @@ storage_account_key: {{storage_account_key}} {{#if resource_manager_endpoint}} resource_manager_endpoint: {{resource_manager_endpoint}} {{/if}} +{{#if transport}} +transport: {{transport}} +{{/if}} tags: {{#if preserve_original_event}} - preserve_original_event diff --git a/packages/azure_app_service/docs/README.md b/packages/azure_app_service/docs/README.md index a71fafe3d6c..f0154069f06 100644 --- a/packages/azure_app_service/docs/README.md +++ b/packages/azure_app_service/docs/README.md @@ -23,6 +23,12 @@ Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for m **Authentication (Event Hub):** The Event Hub input supports two authentication methods: **connection string** (default) and **client secret** (Microsoft Entra ID). For setup steps, required RBAC roles (Azure Event Hubs Data Receiver, Storage Blob Data Contributor), and configuration options, see the [Azure Logs integration](https://docs.elastic.co/integrations/azure) or [Filebeat azure-eventhub input](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-eventhub.html) documentation. +### AMQP-over-WebSockets and proxy support + +By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. + +This option requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. + ## App Service Logs Collects different types of logs from Azure App Service via Event Hub. diff --git a/packages/azure_app_service/manifest.yml b/packages/azure_app_service/manifest.yml index 889e7b5c9ef..7e9b739a895 100644 --- a/packages/azure_app_service/manifest.yml +++ b/packages/azure_app_service/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: azure_app_service title: "Azure App Service" -version: "1.1.0" +version: "1.2.0" source: license: "Elastic-2.0" description: "Collect logs from Azure App Service with Elastic Agent." @@ -131,6 +131,21 @@ vars: multi: false required: false show_user: false + - name: transport + type: select + title: Event Hubs transport protocol + multi: false + required: true + show_user: false + default: amqp + options: + - text: AMQP + value: amqp + - text: AMQP-over-WebSockets + value: websocket + description: > + (Processor v2 only) The transport protocol used to connect to Event Hubs. + Supported values are AMQP and AMQP-over-WebSockets. Defaults to AMQP. icons: - src: /img/app-service-logo.svg title: App Service Logo diff --git a/packages/azure_functions/_dev/build/docs/README.md b/packages/azure_functions/_dev/build/docs/README.md index 756f695f46e..9760168df21 100644 --- a/packages/azure_functions/_dev/build/docs/README.md +++ b/packages/azure_functions/_dev/build/docs/README.md @@ -96,6 +96,12 @@ Instead of a connection string, you can authenticate using a Microsoft Entra ID 4. **Configure the integration** Set **Authentication type** to **Client Secret**. Provide **Tenant ID**, **Client ID**, **Client Secret**, and the fully qualified **Event Hub namespace** (for example `yournamespace.servicebus.windows.net`). Use the same Storage Account and container as for connection string authentication; the integration will use the client secret to access both Event Hubs and Storage. +#### AMQP-over-WebSockets and proxy support + +By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. + +This option requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. + #### Configuration options `auth_type` : @@ -144,6 +150,10 @@ _string_ _string_ (Optional, for client secret authentication.) Microsoft Entra ID authority endpoint. Defaults to `https://login.microsoftonline.com` (Azure Public Cloud). Use a different endpoint for other clouds (for example Azure Government, China, Germany). +`transport` : +_string_ +(Processor v2 only) The transport protocol to use when connecting to Event Hubs. `amqp` (default) uses ports `5671` and `5672`. `websocket` uses AMQP-over-WebSockets on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. + `storage_account_container` : _string_ The storage account container where the integration stores the checkpoint data for the consumer group. It is an advanced option to use with extreme care. You must use a dedicated storage account container for each Azure log type (activity, sign-in, audit logs, and others). Do not reuse the same container name for more than one Azure log type. Refer to [Container Names](https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names) for details on naming rules from Microsoft. The integration generates a default container name, if not specified. diff --git a/packages/azure_functions/changelog.yml b/packages/azure_functions/changelog.yml index 6777971173b..2168c210846 100644 --- a/packages/azure_functions/changelog.yml +++ b/packages/azure_functions/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.16.0" + changes: + - description: Add support for AMQP-over-WebSockets transport protocol and proxy configuration. + type: enhancement + link: https://github.com/elastic/obs-integration-team/issues/973 - version: "0.15.0" changes: - description: Publish Agentless to Elastic Managed integrations name change. diff --git a/packages/azure_functions/data_stream/functionapplogs/agent/stream/azure-eventhub.yml.hbs b/packages/azure_functions/data_stream/functionapplogs/agent/stream/azure-eventhub.yml.hbs index d0b31ea0389..cd50f199835 100644 --- a/packages/azure_functions/data_stream/functionapplogs/agent/stream/azure-eventhub.yml.hbs +++ b/packages/azure_functions/data_stream/functionapplogs/agent/stream/azure-eventhub.yml.hbs @@ -43,6 +43,9 @@ storage_account_key: {{storage_account_key}} {{#if resource_manager_endpoint}} resource_manager_endpoint: {{resource_manager_endpoint}} {{/if}} +{{#if transport}} +transport: {{transport}} +{{/if}} tags: {{#if preserve_original_event}} - preserve_original_event diff --git a/packages/azure_functions/data_stream/functionapplogs/manifest.yml b/packages/azure_functions/data_stream/functionapplogs/manifest.yml index 30b022b43ea..3a97a9da9e0 100644 --- a/packages/azure_functions/data_stream/functionapplogs/manifest.yml +++ b/packages/azure_functions/data_stream/functionapplogs/manifest.yml @@ -114,6 +114,21 @@ streams: description: >- (Optional when **Authentication Type** is **Client Secret**) Microsoft Entra ID authority endpoint. Defaults to https://login.microsoftonline.com (Azure Public Cloud). Change for other Azure environments: Azure Government (https://login.microsoftonline.us), Azure China (https://login.chinacloudapi.cn), or Azure Germany (https://login.microsoftonline.de). + - name: transport + type: select + title: Event Hubs transport protocol + multi: false + required: true + show_user: false + default: amqp + options: + - text: AMQP + value: amqp + - text: AMQP-over-WebSockets + value: websocket + description: > + (Processor v2 only) The transport protocol used to connect to Event Hubs. + Supported values are AMQP and AMQP-over-WebSockets. Defaults to AMQP. - name: preserve_original_event required: true show_user: true diff --git a/packages/azure_functions/docs/README.md b/packages/azure_functions/docs/README.md index 11285ac8518..2cfbc0edd5f 100644 --- a/packages/azure_functions/docs/README.md +++ b/packages/azure_functions/docs/README.md @@ -96,6 +96,12 @@ Instead of a connection string, you can authenticate using a Microsoft Entra ID 4. **Configure the integration** Set **Authentication type** to **Client Secret**. Provide **Tenant ID**, **Client ID**, **Client Secret**, and the fully qualified **Event Hub namespace** (for example `yournamespace.servicebus.windows.net`). Use the same Storage Account and container as for connection string authentication; the integration will use the client secret to access both Event Hubs and Storage. +#### AMQP-over-WebSockets and proxy support + +By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. + +This option requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. + #### Configuration options `auth_type` : @@ -144,6 +150,10 @@ _string_ _string_ (Optional, for client secret authentication.) Microsoft Entra ID authority endpoint. Defaults to `https://login.microsoftonline.com` (Azure Public Cloud). Use a different endpoint for other clouds (for example Azure Government, China, Germany). +`transport` : +_string_ +(Processor v2 only) The transport protocol to use when connecting to Event Hubs. `amqp` (default) uses ports `5671` and `5672`. `websocket` uses AMQP-over-WebSockets on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. + `storage_account_container` : _string_ The storage account container where the integration stores the checkpoint data for the consumer group. It is an advanced option to use with extreme care. You must use a dedicated storage account container for each Azure log type (activity, sign-in, audit logs, and others). Do not reuse the same container name for more than one Azure log type. Refer to [Container Names](https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names) for details on naming rules from Microsoft. The integration generates a default container name, if not specified. diff --git a/packages/azure_functions/manifest.yml b/packages/azure_functions/manifest.yml index 13ce474ee73..0fe5043eee5 100644 --- a/packages/azure_functions/manifest.yml +++ b/packages/azure_functions/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.3.2" name: azure_functions title: "Azure Functions" -version: "0.15.0" +version: "0.16.0" source: license: "Elastic-2.0" description: "Get metrics and logs from Azure Functions" diff --git a/packages/azure_logs/_dev/build/docs/README.md b/packages/azure_logs/_dev/build/docs/README.md index c8ce0f4b3a0..bec48226380 100644 --- a/packages/azure_logs/_dev/build/docs/README.md +++ b/packages/azure_logs/_dev/build/docs/README.md @@ -328,7 +328,9 @@ The Agent creates one SA container for the integration. The SA container name co ### Running the integration behind a firewall -When you run the Elastic Agent behind a firewall, you must allow traffic on ports `5671` and `5672` for the event hub and port `443` for the Storage Account container to ensure proper communication with the necessary components. +When the Elastic Agent runs in an environment with network restrictions, check that the required ports are open for the transport protocol used by the integration. + +The Elastic Agent requires access to Event Hubs and Storage Accounts. ```text ┌────────────────────────────────┐ ┌───────────────────┐ ┌───────────────────┐ @@ -356,18 +358,26 @@ When you run the Elastic Agent behind a firewall, you must allow traffic on port └─Azure──────────────────────────┘ ``` -#### Event hub +#### Event Hubs (AMQP) + +By default, the integration uses AMQP to communicate with Event Hubs. -Port `5671` and `5672` are commonly used for secure communication with the event hub. These ports are used to receive events. The Elastic Agent can establish a secure connection with the event hub by allowing traffic on these ports. +AMQP uses ports `5671` and `5672`. The Elastic Agent initiates outbound TCP connections to these ports on the Azure Event Hubs service to receive events. For more information, check the following documents: -* [What ports do I need to open on the firewall?](https://learn.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-ports-do-i-need-to-open-on-the-firewall) from the [Event Hubs frequently asked questions](https://learn.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-ports-do-i-need-to-open-on-the-firewall). -* [AMQP outbound port requirements](https://learn.microsoft.com/en-us/azure/service-bus-messaging/service-bus-amqp-protocol-guide#amqp-outbound-port-requirements) +- [What ports do I need to open on the firewall?](https://learn.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-ports-do-i-need-to-open-on-the-firewall) from the [Event Hubs frequently asked questions](https://learn.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-ports-do-i-need-to-open-on-the-firewall). +- [AMQP outbound port requirements](https://learn.microsoft.com/en-us/azure/service-bus-messaging/service-bus-amqp-protocol-guide#amqp-outbound-port-requirements) + +#### Event Hubs (AMQP-over-WebSockets) -#### Storage Account container +If ports `5671` and `5672` are blocked, the integration can use AMQP-over-WebSockets. This protocol tunnels AMQP over port `443` (HTTPS), which is typically allowed through firewalls. -Port `443` is used for secure communication with the Storage Account container. This port is commonly used for HTTPS traffic. By allowing traffic on port 443, the Elastic Agent can securely access and interact with the Storage Account container, essential for storing and retrieving checkpoint data for each event hub partition. +To use it, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. + +#### Storage Account + +The Elastic Agent initiates outbound TCP connections to port `443` (HTTPS) to store and retrieve checkpoint data from the Azure Storage Account service. #### DNS @@ -379,6 +389,12 @@ Optionally, you can restrict the traffic to the following domain names: *.cloudapp.net ``` +#### Proxy support + +When using AMQP-over-WebSockets, both Event Hubs and Storage Account traffic use HTTPS on port `443`. Set the `HTTPS_PROXY` environment variable to route this traffic through a proxy. + +Proxy support requires **Event Hubs transport protocol** set to **AMQP-over-WebSockets**, processor v2, and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. + ## Settings Use the following settings to configure the Azure Logs integration when you add it to Fleet. @@ -492,6 +508,15 @@ _string_ The partition consumer waits up to a "receive count" or a "receive timeout", whichever comes first. Default is `100` messages. +`transport` : +_string_ +(processor v2 only) The transport protocol to use when connecting to Event Hubs. Possible values are: + +* `amqp` (default): Use AMQP on ports `5671` and `5672`. +* `websocket`: Use AMQP-over-WebSockets on port `443`. Use this option when AMQP ports are blocked. + +When `websocket` is selected, traffic can be routed through a proxy by setting the `HTTPS_PROXY` environment variable. + ## Handling Malformed JSON in Azure Logs Azure services have been observed occasionally sending [malformed JSON](https://learn.microsoft.com/en-us/answers/questions/1001797/invalid-json-logs-produced-for-function-apps) documents. These logs can disrupt the expected JSON formatting and lead to parsing issues during processing. diff --git a/packages/azure_logs/agent/input/input.yml.hbs b/packages/azure_logs/agent/input/input.yml.hbs index 72f94fafe7a..c5050892995 100644 --- a/packages/azure_logs/agent/input/input.yml.hbs +++ b/packages/azure_logs/agent/input/input.yml.hbs @@ -68,6 +68,9 @@ partition_receive_timeout: {{partition_receive_timeout}} {{#if partition_receive_count}} partition_receive_count: {{partition_receive_count}} {{/if}} +{{#if transport}} +transport: {{transport}} +{{/if}} tags: {{#if preserve_original_event}} @@ -89,4 +92,4 @@ sanitize_options: {{/if}} {{#if sanitize_singlequotes}} - SINGLE_QUOTES -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/azure_logs/changelog.yml b/packages/azure_logs/changelog.yml index bfbc3cbb147..55644fc8d1b 100644 --- a/packages/azure_logs/changelog.yml +++ b/packages/azure_logs/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.7.0" + changes: + - description: Add support for AMQP-over-WebSockets transport protocol and proxy configuration. + type: enhancement + link: https://github.com/elastic/obs-integration-team/issues/973 - version: "0.6.0" changes: - description: Add client secret authentication support for Azure Event Hub with RBAC. diff --git a/packages/azure_logs/docs/README.md b/packages/azure_logs/docs/README.md index c8ce0f4b3a0..bec48226380 100644 --- a/packages/azure_logs/docs/README.md +++ b/packages/azure_logs/docs/README.md @@ -328,7 +328,9 @@ The Agent creates one SA container for the integration. The SA container name co ### Running the integration behind a firewall -When you run the Elastic Agent behind a firewall, you must allow traffic on ports `5671` and `5672` for the event hub and port `443` for the Storage Account container to ensure proper communication with the necessary components. +When the Elastic Agent runs in an environment with network restrictions, check that the required ports are open for the transport protocol used by the integration. + +The Elastic Agent requires access to Event Hubs and Storage Accounts. ```text ┌────────────────────────────────┐ ┌───────────────────┐ ┌───────────────────┐ @@ -356,18 +358,26 @@ When you run the Elastic Agent behind a firewall, you must allow traffic on port └─Azure──────────────────────────┘ ``` -#### Event hub +#### Event Hubs (AMQP) + +By default, the integration uses AMQP to communicate with Event Hubs. -Port `5671` and `5672` are commonly used for secure communication with the event hub. These ports are used to receive events. The Elastic Agent can establish a secure connection with the event hub by allowing traffic on these ports. +AMQP uses ports `5671` and `5672`. The Elastic Agent initiates outbound TCP connections to these ports on the Azure Event Hubs service to receive events. For more information, check the following documents: -* [What ports do I need to open on the firewall?](https://learn.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-ports-do-i-need-to-open-on-the-firewall) from the [Event Hubs frequently asked questions](https://learn.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-ports-do-i-need-to-open-on-the-firewall). -* [AMQP outbound port requirements](https://learn.microsoft.com/en-us/azure/service-bus-messaging/service-bus-amqp-protocol-guide#amqp-outbound-port-requirements) +- [What ports do I need to open on the firewall?](https://learn.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-ports-do-i-need-to-open-on-the-firewall) from the [Event Hubs frequently asked questions](https://learn.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-ports-do-i-need-to-open-on-the-firewall). +- [AMQP outbound port requirements](https://learn.microsoft.com/en-us/azure/service-bus-messaging/service-bus-amqp-protocol-guide#amqp-outbound-port-requirements) + +#### Event Hubs (AMQP-over-WebSockets) -#### Storage Account container +If ports `5671` and `5672` are blocked, the integration can use AMQP-over-WebSockets. This protocol tunnels AMQP over port `443` (HTTPS), which is typically allowed through firewalls. -Port `443` is used for secure communication with the Storage Account container. This port is commonly used for HTTPS traffic. By allowing traffic on port 443, the Elastic Agent can securely access and interact with the Storage Account container, essential for storing and retrieving checkpoint data for each event hub partition. +To use it, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. + +#### Storage Account + +The Elastic Agent initiates outbound TCP connections to port `443` (HTTPS) to store and retrieve checkpoint data from the Azure Storage Account service. #### DNS @@ -379,6 +389,12 @@ Optionally, you can restrict the traffic to the following domain names: *.cloudapp.net ``` +#### Proxy support + +When using AMQP-over-WebSockets, both Event Hubs and Storage Account traffic use HTTPS on port `443`. Set the `HTTPS_PROXY` environment variable to route this traffic through a proxy. + +Proxy support requires **Event Hubs transport protocol** set to **AMQP-over-WebSockets**, processor v2, and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. + ## Settings Use the following settings to configure the Azure Logs integration when you add it to Fleet. @@ -492,6 +508,15 @@ _string_ The partition consumer waits up to a "receive count" or a "receive timeout", whichever comes first. Default is `100` messages. +`transport` : +_string_ +(processor v2 only) The transport protocol to use when connecting to Event Hubs. Possible values are: + +* `amqp` (default): Use AMQP on ports `5671` and `5672`. +* `websocket`: Use AMQP-over-WebSockets on port `443`. Use this option when AMQP ports are blocked. + +When `websocket` is selected, traffic can be routed through a proxy by setting the `HTTPS_PROXY` environment variable. + ## Handling Malformed JSON in Azure Logs Azure services have been observed occasionally sending [malformed JSON](https://learn.microsoft.com/en-us/answers/questions/1001797/invalid-json-logs-produced-for-function-apps) documents. These logs can disrupt the expected JSON formatting and lead to parsing issues during processing. diff --git a/packages/azure_logs/manifest.yml b/packages/azure_logs/manifest.yml index 0747419c2f1..c5bf721209e 100644 --- a/packages/azure_logs/manifest.yml +++ b/packages/azure_logs/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.0 name: azure_logs title: "Custom Azure Logs" -version: "0.6.0" +version: "0.7.0" source: license: Elastic-2.0 description: "Collect log events from Azure Event Hubs with Elastic Agent" @@ -277,6 +277,21 @@ policy_templates: The partition consumer waits up to a "receive count" or a "receive timeout", whichever comes first. Default is `100` messages. + - name: transport + type: select + title: Event Hubs transport protocol + multi: false + required: true + show_user: false + default: amqp + options: + - text: AMQP + value: amqp + - text: AMQP-over-WebSockets + value: websocket + description: > + (Processor v2 only) The transport protocol used to connect to Event Hubs. + Supported values are AMQP and AMQP-over-WebSockets. Defaults to AMQP. - name: migrate_checkpoint type: bool title: Migrate checkpoint information diff --git a/packages/azure_openai/_dev/build/docs/README.md b/packages/azure_openai/_dev/build/docs/README.md index c5e8919d3d0..faa90dfaff2 100644 --- a/packages/azure_openai/_dev/build/docs/README.md +++ b/packages/azure_openai/_dev/build/docs/README.md @@ -41,6 +41,12 @@ Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for m **Authentication (Event Hub):** The Event Hub input supports two authentication methods: **connection string** (default) and **client secret** (Microsoft Entra ID). For setup steps, required RBAC roles (Azure Event Hubs Data Receiver, Storage Blob Data Contributor), and configuration options, see the [Azure Logs integration](https://docs.elastic.co/integrations/azure) or [Filebeat azure-eventhub input](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-eventhub.html) documentation. +#### AMQP-over-WebSockets and proxy support + +By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. + +This option requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. + #### Default Logging The Azure OpenAI provides native logging and monitoring to track the telemetry of the service. The Audit and RequestResponse log categories come under the native logging. However, the default logging doesn't log the inputs and outputs of the service. This is useful to ensure that the services operates as expected. @@ -129,4 +135,4 @@ For more details on ECS fields, check the [ECS Field Reference](https://www.elas {{fields "metrics"}} ## Alerting Rule Template -{{alertRuleTemplates}} \ No newline at end of file +{{alertRuleTemplates}} diff --git a/packages/azure_openai/changelog.yml b/packages/azure_openai/changelog.yml index 27d0548f4ed..bb284d4e37e 100644 --- a/packages/azure_openai/changelog.yml +++ b/packages/azure_openai/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.15.0" + changes: + - description: Add support for AMQP-over-WebSockets transport protocol and proxy configuration. + type: enhancement + link: https://github.com/elastic/obs-integration-team/issues/973 - version: "1.14.0" changes: - description: Publish Agentless to Elastic Managed integrations name change. diff --git a/packages/azure_openai/data_stream/logs/agent/stream/azure-eventhub.yml.hbs b/packages/azure_openai/data_stream/logs/agent/stream/azure-eventhub.yml.hbs index 048e70583c2..197530b398c 100644 --- a/packages/azure_openai/data_stream/logs/agent/stream/azure-eventhub.yml.hbs +++ b/packages/azure_openai/data_stream/logs/agent/stream/azure-eventhub.yml.hbs @@ -43,6 +43,9 @@ storage_account_key: {{storage_account_key}} {{#if resource_manager_endpoint}} resource_manager_endpoint: {{resource_manager_endpoint}} {{/if}} +{{#if transport}} +transport: {{transport}} +{{/if}} tags: {{#if preserve_original_event}} - preserve_original_event diff --git a/packages/azure_openai/data_stream/logs/manifest.yml b/packages/azure_openai/data_stream/logs/manifest.yml index bc36c0d6811..824fc96ee4f 100644 --- a/packages/azure_openai/data_stream/logs/manifest.yml +++ b/packages/azure_openai/data_stream/logs/manifest.yml @@ -113,6 +113,21 @@ streams: description: >- (Optional when **Authentication Type** is **Client Secret**) Microsoft Entra ID authority endpoint. Defaults to https://login.microsoftonline.com (Azure Public Cloud). Change for other Azure environments: Azure Government (https://login.microsoftonline.us), Azure China (https://login.chinacloudapi.cn), or Azure Germany (https://login.microsoftonline.de). + - name: transport + type: select + title: Event Hubs transport protocol + multi: false + required: true + show_user: false + default: amqp + options: + - text: AMQP + value: amqp + - text: AMQP-over-WebSockets + value: websocket + description: > + (Processor v2 only) The transport protocol used to connect to Event Hubs. + Supported values are AMQP and AMQP-over-WebSockets. Defaults to AMQP. - name: preserve_original_event required: true show_user: true diff --git a/packages/azure_openai/docs/README.md b/packages/azure_openai/docs/README.md index 1d2fb8fa8c6..aa07b9913a0 100644 --- a/packages/azure_openai/docs/README.md +++ b/packages/azure_openai/docs/README.md @@ -41,6 +41,12 @@ Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for m **Authentication (Event Hub):** The Event Hub input supports two authentication methods: **connection string** (default) and **client secret** (Microsoft Entra ID). For setup steps, required RBAC roles (Azure Event Hubs Data Receiver, Storage Blob Data Contributor), and configuration options, see the [Azure Logs integration](https://docs.elastic.co/integrations/azure) or [Filebeat azure-eventhub input](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-eventhub.html) documentation. +#### AMQP-over-WebSockets and proxy support + +By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. + +This option requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. + #### Default Logging The Azure OpenAI provides native logging and monitoring to track the telemetry of the service. The Audit and RequestResponse log categories come under the native logging. However, the default logging doesn't log the inputs and outputs of the service. This is useful to ensure that the services operates as expected. @@ -418,4 +424,3 @@ The following alert rule templates are available: **[Azure OpenAI] Quota Error Rates above threshold** - diff --git a/packages/azure_openai/manifest.yml b/packages/azure_openai/manifest.yml index 1f6d9f2148e..b8cec6d2d11 100644 --- a/packages/azure_openai/manifest.yml +++ b/packages/azure_openai/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.4.0 name: azure_openai title: "Azure OpenAI" -version: "1.14.0" +version: "1.15.0" source: license: "Elastic-2.0" description: "Collects Azure OpenAI Logs and Metrics" From 654c263b21cbef8e3e5a0355c9c5d434da801693 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Wed, 5 Aug 2026 20:12:21 +0200 Subject: [PATCH 2/5] [Azure] Link changelogs to pull request --- packages/azure_ai_foundry/changelog.yml | 2 +- packages/azure_app_service/changelog.yml | 2 +- packages/azure_functions/changelog.yml | 2 +- packages/azure_logs/changelog.yml | 2 +- packages/azure_openai/changelog.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/azure_ai_foundry/changelog.yml b/packages/azure_ai_foundry/changelog.yml index e2e495c8043..15b3dea10d3 100644 --- a/packages/azure_ai_foundry/changelog.yml +++ b/packages/azure_ai_foundry/changelog.yml @@ -2,7 +2,7 @@ changes: - description: Add support for AMQP-over-WebSockets transport protocol and proxy configuration. type: enhancement - link: https://github.com/elastic/obs-integration-team/issues/973 + link: https://github.com/elastic/integrations/pull/20559 - version: "0.12.0" changes: - description: Publish Agentless to Elastic Managed integrations name change. diff --git a/packages/azure_app_service/changelog.yml b/packages/azure_app_service/changelog.yml index df89e4f875c..e1bacddf651 100644 --- a/packages/azure_app_service/changelog.yml +++ b/packages/azure_app_service/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add support for AMQP-over-WebSockets transport protocol and proxy configuration. type: enhancement - link: https://github.com/elastic/obs-integration-team/issues/973 + link: https://github.com/elastic/integrations/pull/20559 - version: "1.1.0" changes: - description: Add client secret authentication support for Azure Event Hub with RBAC. diff --git a/packages/azure_functions/changelog.yml b/packages/azure_functions/changelog.yml index 2168c210846..b106ad6dd48 100644 --- a/packages/azure_functions/changelog.yml +++ b/packages/azure_functions/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add support for AMQP-over-WebSockets transport protocol and proxy configuration. type: enhancement - link: https://github.com/elastic/obs-integration-team/issues/973 + link: https://github.com/elastic/integrations/pull/20559 - version: "0.15.0" changes: - description: Publish Agentless to Elastic Managed integrations name change. diff --git a/packages/azure_logs/changelog.yml b/packages/azure_logs/changelog.yml index 55644fc8d1b..4a4c7164c12 100644 --- a/packages/azure_logs/changelog.yml +++ b/packages/azure_logs/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add support for AMQP-over-WebSockets transport protocol and proxy configuration. type: enhancement - link: https://github.com/elastic/obs-integration-team/issues/973 + link: https://github.com/elastic/integrations/pull/20559 - version: "0.6.0" changes: - description: Add client secret authentication support for Azure Event Hub with RBAC. diff --git a/packages/azure_openai/changelog.yml b/packages/azure_openai/changelog.yml index bb284d4e37e..1be25fe1b5d 100644 --- a/packages/azure_openai/changelog.yml +++ b/packages/azure_openai/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add support for AMQP-over-WebSockets transport protocol and proxy configuration. type: enhancement - link: https://github.com/elastic/obs-integration-team/issues/973 + link: https://github.com/elastic/integrations/pull/20559 - version: "1.14.0" changes: - description: Publish Agentless to Elastic Managed integrations name change. From 3fd6274802aaf67df37261a377735d78da2a5dbe Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Thu, 6 Aug 2026 18:48:50 +0200 Subject: [PATCH 3/5] Expose Event Hub processor version so the transport option takes effect The new `transport` option is read only by the v2 Event Hub processor (`CreateEventHubConsumerClient` is called exclusively from `v2_input.go`; the v1 input uses the legacy azure-event-hubs-go SDK and ignores it). azure_ai_foundry, azure_openai, azure_functions and azure_app_service never emitted `processor_version`, so Beats applied its default, which is v1 on 8.19, 9.1 and 9.2. Selecting AMQP-over-WebSockets on those versions silently kept dialing AMQP on 5671/5672. Add a `processor_version` var defaulting to v2 and emit it from the agent templates, matching azure and azure_logs. Also fix the docs: name the processor version requirement explicitly and correct the supported agent versions per package (azure_ai_foundry cannot be installed before 9.2.4, azure_openai has no 9.1.x). Co-Authored-By: Claude Opus 5 --- .../azure_ai_foundry/_dev/build/docs/README.md | 2 +- packages/azure_ai_foundry/changelog.yml | 3 +++ .../logs/agent/stream/azure-eventhub.yml.hbs | 3 +++ .../data_stream/logs/manifest.yml | 15 +++++++++++++++ packages/azure_ai_foundry/docs/README.md | 4 +++- .../azure_app_service/_dev/build/docs/README.md | 2 +- packages/azure_app_service/changelog.yml | 3 +++ .../agent/stream/azure-eventhub.yml.hbs | 3 +++ packages/azure_app_service/docs/README.md | 2 +- packages/azure_app_service/manifest.yml | 15 +++++++++++++++ .../azure_functions/_dev/build/docs/README.md | 6 +++++- packages/azure_functions/changelog.yml | 3 +++ .../agent/stream/azure-eventhub.yml.hbs | 3 +++ .../data_stream/functionapplogs/manifest.yml | 15 +++++++++++++++ packages/azure_functions/docs/README.md | 6 +++++- packages/azure_openai/_dev/build/docs/README.md | 2 +- packages/azure_openai/changelog.yml | 3 +++ .../logs/agent/stream/azure-eventhub.yml.hbs | 3 +++ .../azure_openai/data_stream/logs/manifest.yml | 15 +++++++++++++++ packages/azure_openai/docs/README.md | 4 +++- 20 files changed, 104 insertions(+), 8 deletions(-) diff --git a/packages/azure_ai_foundry/_dev/build/docs/README.md b/packages/azure_ai_foundry/_dev/build/docs/README.md index 3927357c457..9a398526511 100644 --- a/packages/azure_ai_foundry/_dev/build/docs/README.md +++ b/packages/azure_ai_foundry/_dev/build/docs/README.md @@ -40,7 +40,7 @@ Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for m By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. -This option requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. +This option requires the Event Hub processor v2 (**Processor version** set to `v2`, which is the default) and Elastic Agent 9.2.4 or later. #### Native logging diff --git a/packages/azure_ai_foundry/changelog.yml b/packages/azure_ai_foundry/changelog.yml index 15b3dea10d3..9e04c190d92 100644 --- a/packages/azure_ai_foundry/changelog.yml +++ b/packages/azure_ai_foundry/changelog.yml @@ -3,6 +3,9 @@ - description: Add support for AMQP-over-WebSockets transport protocol and proxy configuration. type: enhancement link: https://github.com/elastic/integrations/pull/20559 + - description: Add the Event Hub processor version option and switch the default processor version to v2. + type: enhancement + link: https://github.com/elastic/integrations/pull/20559 - version: "0.12.0" changes: - description: Publish Agentless to Elastic Managed integrations name change. diff --git a/packages/azure_ai_foundry/data_stream/logs/agent/stream/azure-eventhub.yml.hbs b/packages/azure_ai_foundry/data_stream/logs/agent/stream/azure-eventhub.yml.hbs index 4b0a538ac48..c490d2e45e1 100644 --- a/packages/azure_ai_foundry/data_stream/logs/agent/stream/azure-eventhub.yml.hbs +++ b/packages/azure_ai_foundry/data_stream/logs/agent/stream/azure-eventhub.yml.hbs @@ -43,6 +43,9 @@ storage_account_key: {{storage_account_key}} {{#if resource_manager_endpoint}} resource_manager_endpoint: {{resource_manager_endpoint}} {{/if}} +{{#if processor_version}} +processor_version: {{processor_version}} +{{/if}} {{#if transport}} transport: {{transport}} {{/if}} diff --git a/packages/azure_ai_foundry/data_stream/logs/manifest.yml b/packages/azure_ai_foundry/data_stream/logs/manifest.yml index 3a54804c267..182bb34daf9 100644 --- a/packages/azure_ai_foundry/data_stream/logs/manifest.yml +++ b/packages/azure_ai_foundry/data_stream/logs/manifest.yml @@ -113,6 +113,21 @@ streams: description: >- (Optional when **Authentication Type** is **Client Secret**) Microsoft Entra ID authority endpoint. Defaults to https://login.microsoftonline.com (Azure Public Cloud). Change for other Azure environments: Azure Government (https://login.microsoftonline.us), Azure China (https://login.chinacloudapi.cn), or Azure Germany (https://login.microsoftonline.de). + - name: processor_version + type: select + title: Processor version + multi: false + required: false + show_user: false + default: v2 + options: + - text: v1 (legacy) + value: v1 + - text: v2 + value: v2 + description: > + The processor version that the integration should use. Possible values are v1 (legacy) and v2 (recommended). + The v2 event hub processor is recommended for typical use cases. Defaults to v2. - name: transport type: select title: Event Hubs transport protocol diff --git a/packages/azure_ai_foundry/docs/README.md b/packages/azure_ai_foundry/docs/README.md index af644c91746..3f065e9e83a 100644 --- a/packages/azure_ai_foundry/docs/README.md +++ b/packages/azure_ai_foundry/docs/README.md @@ -40,7 +40,7 @@ Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for m By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. -This option requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. +This option requires the Event Hub processor v2 (**Processor version** set to `v2`, which is the default) and Elastic Agent 9.2.4 or later. #### Native logging @@ -509,3 +509,5 @@ The following alert rule templates are available: **[Microsoft Foundry] Provisioned Utilization above threshold** + + diff --git a/packages/azure_app_service/_dev/build/docs/README.md b/packages/azure_app_service/_dev/build/docs/README.md index 9ccf44e717c..4fefd1c64f0 100644 --- a/packages/azure_app_service/_dev/build/docs/README.md +++ b/packages/azure_app_service/_dev/build/docs/README.md @@ -27,7 +27,7 @@ Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for m By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. -This option requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. +This option requires the Event Hub processor v2 (**Processor version** set to `v2`, which is the default) and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. ## App Service Logs diff --git a/packages/azure_app_service/changelog.yml b/packages/azure_app_service/changelog.yml index e1bacddf651..1c1286cf766 100644 --- a/packages/azure_app_service/changelog.yml +++ b/packages/azure_app_service/changelog.yml @@ -4,6 +4,9 @@ - description: Add support for AMQP-over-WebSockets transport protocol and proxy configuration. type: enhancement link: https://github.com/elastic/integrations/pull/20559 + - description: Add the Event Hub processor version option and switch the default processor version to v2. + type: enhancement + link: https://github.com/elastic/integrations/pull/20559 - version: "1.1.0" changes: - description: Add client secret authentication support for Azure Event Hub with RBAC. diff --git a/packages/azure_app_service/data_stream/app_service_logs/agent/stream/azure-eventhub.yml.hbs b/packages/azure_app_service/data_stream/app_service_logs/agent/stream/azure-eventhub.yml.hbs index 4e8518c49ed..b90090afe52 100644 --- a/packages/azure_app_service/data_stream/app_service_logs/agent/stream/azure-eventhub.yml.hbs +++ b/packages/azure_app_service/data_stream/app_service_logs/agent/stream/azure-eventhub.yml.hbs @@ -43,6 +43,9 @@ storage_account_key: {{storage_account_key}} {{#if resource_manager_endpoint}} resource_manager_endpoint: {{resource_manager_endpoint}} {{/if}} +{{#if processor_version}} +processor_version: {{processor_version}} +{{/if}} {{#if transport}} transport: {{transport}} {{/if}} diff --git a/packages/azure_app_service/docs/README.md b/packages/azure_app_service/docs/README.md index f0154069f06..84cf75a934d 100644 --- a/packages/azure_app_service/docs/README.md +++ b/packages/azure_app_service/docs/README.md @@ -27,7 +27,7 @@ Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for m By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. -This option requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. +This option requires the Event Hub processor v2 (**Processor version** set to `v2`, which is the default) and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. ## App Service Logs diff --git a/packages/azure_app_service/manifest.yml b/packages/azure_app_service/manifest.yml index 7e9b739a895..d7489dec262 100644 --- a/packages/azure_app_service/manifest.yml +++ b/packages/azure_app_service/manifest.yml @@ -131,6 +131,21 @@ vars: multi: false required: false show_user: false + - name: processor_version + type: select + title: Processor version + multi: false + required: false + show_user: false + default: v2 + options: + - text: v1 (legacy) + value: v1 + - text: v2 + value: v2 + description: > + The processor version that the integration should use. Possible values are v1 (legacy) and v2 (recommended). + The v2 event hub processor is recommended for typical use cases. Defaults to v2. - name: transport type: select title: Event Hubs transport protocol diff --git a/packages/azure_functions/_dev/build/docs/README.md b/packages/azure_functions/_dev/build/docs/README.md index 9760168df21..43b91fd1143 100644 --- a/packages/azure_functions/_dev/build/docs/README.md +++ b/packages/azure_functions/_dev/build/docs/README.md @@ -100,7 +100,7 @@ Instead of a connection string, you can authenticate using a Microsoft Entra ID By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. -This option requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. +This option requires the Event Hub processor v2 (**Processor version** set to `v2`, which is the default) and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. #### Configuration options @@ -150,6 +150,10 @@ _string_ _string_ (Optional, for client secret authentication.) Microsoft Entra ID authority endpoint. Defaults to `https://login.microsoftonline.com` (Azure Public Cloud). Use a different endpoint for other clouds (for example Azure Government, China, Germany). +`processor_version` : +_string_ +The Event Hub processor version that the integration should use. Possible values are `v1` (legacy) and `v2` (recommended). Defaults to `v2`. + `transport` : _string_ (Processor v2 only) The transport protocol to use when connecting to Event Hubs. `amqp` (default) uses ports `5671` and `5672`. `websocket` uses AMQP-over-WebSockets on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. diff --git a/packages/azure_functions/changelog.yml b/packages/azure_functions/changelog.yml index b106ad6dd48..f3597584136 100644 --- a/packages/azure_functions/changelog.yml +++ b/packages/azure_functions/changelog.yml @@ -4,6 +4,9 @@ - description: Add support for AMQP-over-WebSockets transport protocol and proxy configuration. type: enhancement link: https://github.com/elastic/integrations/pull/20559 + - description: Add the Event Hub processor version option and switch the default processor version to v2. + type: enhancement + link: https://github.com/elastic/integrations/pull/20559 - version: "0.15.0" changes: - description: Publish Agentless to Elastic Managed integrations name change. diff --git a/packages/azure_functions/data_stream/functionapplogs/agent/stream/azure-eventhub.yml.hbs b/packages/azure_functions/data_stream/functionapplogs/agent/stream/azure-eventhub.yml.hbs index cd50f199835..c39b41969d0 100644 --- a/packages/azure_functions/data_stream/functionapplogs/agent/stream/azure-eventhub.yml.hbs +++ b/packages/azure_functions/data_stream/functionapplogs/agent/stream/azure-eventhub.yml.hbs @@ -43,6 +43,9 @@ storage_account_key: {{storage_account_key}} {{#if resource_manager_endpoint}} resource_manager_endpoint: {{resource_manager_endpoint}} {{/if}} +{{#if processor_version}} +processor_version: {{processor_version}} +{{/if}} {{#if transport}} transport: {{transport}} {{/if}} diff --git a/packages/azure_functions/data_stream/functionapplogs/manifest.yml b/packages/azure_functions/data_stream/functionapplogs/manifest.yml index 3a97a9da9e0..32a2855fec2 100644 --- a/packages/azure_functions/data_stream/functionapplogs/manifest.yml +++ b/packages/azure_functions/data_stream/functionapplogs/manifest.yml @@ -114,6 +114,21 @@ streams: description: >- (Optional when **Authentication Type** is **Client Secret**) Microsoft Entra ID authority endpoint. Defaults to https://login.microsoftonline.com (Azure Public Cloud). Change for other Azure environments: Azure Government (https://login.microsoftonline.us), Azure China (https://login.chinacloudapi.cn), or Azure Germany (https://login.microsoftonline.de). + - name: processor_version + type: select + title: Processor version + multi: false + required: false + show_user: false + default: v2 + options: + - text: v1 (legacy) + value: v1 + - text: v2 + value: v2 + description: > + The processor version that the integration should use. Possible values are v1 (legacy) and v2 (recommended). + The v2 event hub processor is recommended for typical use cases. Defaults to v2. - name: transport type: select title: Event Hubs transport protocol diff --git a/packages/azure_functions/docs/README.md b/packages/azure_functions/docs/README.md index 2cfbc0edd5f..f56c56aa56c 100644 --- a/packages/azure_functions/docs/README.md +++ b/packages/azure_functions/docs/README.md @@ -100,7 +100,7 @@ Instead of a connection string, you can authenticate using a Microsoft Entra ID By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. -This option requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. +This option requires the Event Hub processor v2 (**Processor version** set to `v2`, which is the default) and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. #### Configuration options @@ -150,6 +150,10 @@ _string_ _string_ (Optional, for client secret authentication.) Microsoft Entra ID authority endpoint. Defaults to `https://login.microsoftonline.com` (Azure Public Cloud). Use a different endpoint for other clouds (for example Azure Government, China, Germany). +`processor_version` : +_string_ +The Event Hub processor version that the integration should use. Possible values are `v1` (legacy) and `v2` (recommended). Defaults to `v2`. + `transport` : _string_ (Processor v2 only) The transport protocol to use when connecting to Event Hubs. `amqp` (default) uses ports `5671` and `5672`. `websocket` uses AMQP-over-WebSockets on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. diff --git a/packages/azure_openai/_dev/build/docs/README.md b/packages/azure_openai/_dev/build/docs/README.md index faa90dfaff2..6b50d372a2e 100644 --- a/packages/azure_openai/_dev/build/docs/README.md +++ b/packages/azure_openai/_dev/build/docs/README.md @@ -45,7 +45,7 @@ Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for m By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. -This option requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. +This option requires the Event Hub processor v2 (**Processor version** set to `v2`, which is the default) and Elastic Agent 8.19.10, 9.2.4, or later. #### Default Logging diff --git a/packages/azure_openai/changelog.yml b/packages/azure_openai/changelog.yml index 1be25fe1b5d..68cc73cce39 100644 --- a/packages/azure_openai/changelog.yml +++ b/packages/azure_openai/changelog.yml @@ -4,6 +4,9 @@ - description: Add support for AMQP-over-WebSockets transport protocol and proxy configuration. type: enhancement link: https://github.com/elastic/integrations/pull/20559 + - description: Add the Event Hub processor version option and switch the default processor version to v2. + type: enhancement + link: https://github.com/elastic/integrations/pull/20559 - version: "1.14.0" changes: - description: Publish Agentless to Elastic Managed integrations name change. diff --git a/packages/azure_openai/data_stream/logs/agent/stream/azure-eventhub.yml.hbs b/packages/azure_openai/data_stream/logs/agent/stream/azure-eventhub.yml.hbs index 197530b398c..2b5b01dce1a 100644 --- a/packages/azure_openai/data_stream/logs/agent/stream/azure-eventhub.yml.hbs +++ b/packages/azure_openai/data_stream/logs/agent/stream/azure-eventhub.yml.hbs @@ -43,6 +43,9 @@ storage_account_key: {{storage_account_key}} {{#if resource_manager_endpoint}} resource_manager_endpoint: {{resource_manager_endpoint}} {{/if}} +{{#if processor_version}} +processor_version: {{processor_version}} +{{/if}} {{#if transport}} transport: {{transport}} {{/if}} diff --git a/packages/azure_openai/data_stream/logs/manifest.yml b/packages/azure_openai/data_stream/logs/manifest.yml index 824fc96ee4f..0d0ca7f04dc 100644 --- a/packages/azure_openai/data_stream/logs/manifest.yml +++ b/packages/azure_openai/data_stream/logs/manifest.yml @@ -113,6 +113,21 @@ streams: description: >- (Optional when **Authentication Type** is **Client Secret**) Microsoft Entra ID authority endpoint. Defaults to https://login.microsoftonline.com (Azure Public Cloud). Change for other Azure environments: Azure Government (https://login.microsoftonline.us), Azure China (https://login.chinacloudapi.cn), or Azure Germany (https://login.microsoftonline.de). + - name: processor_version + type: select + title: Processor version + multi: false + required: false + show_user: false + default: v2 + options: + - text: v1 (legacy) + value: v1 + - text: v2 + value: v2 + description: > + The processor version that the integration should use. Possible values are v1 (legacy) and v2 (recommended). + The v2 event hub processor is recommended for typical use cases. Defaults to v2. - name: transport type: select title: Event Hubs transport protocol diff --git a/packages/azure_openai/docs/README.md b/packages/azure_openai/docs/README.md index aa07b9913a0..d0bdf019cc0 100644 --- a/packages/azure_openai/docs/README.md +++ b/packages/azure_openai/docs/README.md @@ -45,7 +45,7 @@ Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for m By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. -This option requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. +This option requires the Event Hub processor v2 (**Processor version** set to `v2`, which is the default) and Elastic Agent 8.19.10, 9.2.4, or later. #### Default Logging @@ -424,3 +424,5 @@ The following alert rule templates are available: **[Azure OpenAI] Quota Error Rates above threshold** + + From 43347f07ab56829146a67a579752c2864c383842 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Thu, 6 Aug 2026 18:56:43 +0200 Subject: [PATCH 4/5] Expose the remaining Event Hub processor v2 options Bring azure_ai_foundry, azure_openai, azure_functions and azure_app_service in line with azure and azure_logs by exposing the rest of the processor v2 settings: processor_update_interval, processor_start_position, partition_receive_timeout, partition_receive_count and migrate_checkpoint. migrate_checkpoint is emitted unconditionally rather than behind an `{{#if}}` guard so that setting it to false is honoured; a guarded bool renders nothing when false and the Beats default (true) would apply. endpoint_suffix is deliberately left out: it is not an input option, only a fragment of the storage_account_connection_string that azure and azure_logs assemble by hand. These packages let Beats build that string, and Beats derives the suffix from authority_host, which they already expose. Co-Authored-By: Claude Opus 5 --- packages/azure_ai_foundry/changelog.yml | 3 + .../logs/agent/stream/azure-eventhub.yml.hbs | 13 ++++ .../data_stream/logs/manifest.yml | 72 +++++++++++++++++++ packages/azure_app_service/changelog.yml | 3 + .../agent/stream/azure-eventhub.yml.hbs | 13 ++++ packages/azure_app_service/manifest.yml | 72 +++++++++++++++++++ .../azure_functions/_dev/build/docs/README.md | 20 ++++++ packages/azure_functions/changelog.yml | 3 + .../agent/stream/azure-eventhub.yml.hbs | 13 ++++ .../data_stream/functionapplogs/manifest.yml | 72 +++++++++++++++++++ packages/azure_functions/docs/README.md | 20 ++++++ packages/azure_openai/changelog.yml | 3 + .../logs/agent/stream/azure-eventhub.yml.hbs | 13 ++++ .../data_stream/logs/manifest.yml | 72 +++++++++++++++++++ 14 files changed, 392 insertions(+) diff --git a/packages/azure_ai_foundry/changelog.yml b/packages/azure_ai_foundry/changelog.yml index 9e04c190d92..f855a1a0fd0 100644 --- a/packages/azure_ai_foundry/changelog.yml +++ b/packages/azure_ai_foundry/changelog.yml @@ -6,6 +6,9 @@ - description: Add the Event Hub processor version option and switch the default processor version to v2. type: enhancement link: https://github.com/elastic/integrations/pull/20559 + - description: Add the remaining Event Hub processor v2 options (processor update interval, processor start position, partition receive timeout, partition receive count, and checkpoint migration). + type: enhancement + link: https://github.com/elastic/integrations/pull/20559 - version: "0.12.0" changes: - description: Publish Agentless to Elastic Managed integrations name change. diff --git a/packages/azure_ai_foundry/data_stream/logs/agent/stream/azure-eventhub.yml.hbs b/packages/azure_ai_foundry/data_stream/logs/agent/stream/azure-eventhub.yml.hbs index c490d2e45e1..21f1c959f6f 100644 --- a/packages/azure_ai_foundry/data_stream/logs/agent/stream/azure-eventhub.yml.hbs +++ b/packages/azure_ai_foundry/data_stream/logs/agent/stream/azure-eventhub.yml.hbs @@ -46,6 +46,19 @@ resource_manager_endpoint: {{resource_manager_endpoint}} {{#if processor_version}} processor_version: {{processor_version}} {{/if}} +migrate_checkpoint: {{migrate_checkpoint}} +{{#if processor_update_interval}} +processor_update_interval: {{processor_update_interval}} +{{/if}} +{{#if processor_start_position}} +processor_start_position: {{processor_start_position}} +{{/if}} +{{#if partition_receive_timeout}} +partition_receive_timeout: {{partition_receive_timeout}} +{{/if}} +{{#if partition_receive_count}} +partition_receive_count: {{partition_receive_count}} +{{/if}} {{#if transport}} transport: {{transport}} {{/if}} diff --git a/packages/azure_ai_foundry/data_stream/logs/manifest.yml b/packages/azure_ai_foundry/data_stream/logs/manifest.yml index 182bb34daf9..0e4174c5d00 100644 --- a/packages/azure_ai_foundry/data_stream/logs/manifest.yml +++ b/packages/azure_ai_foundry/data_stream/logs/manifest.yml @@ -128,6 +128,65 @@ streams: description: > The processor version that the integration should use. Possible values are v1 (legacy) and v2 (recommended). The v2 event hub processor is recommended for typical use cases. Defaults to v2. + - name: processor_update_interval + type: text + title: Processor update interval + multi: false + required: false + show_user: false + default: 10s + description: >- + (Processor v2 only) How often the processor should attempt to claim partitions. + + Default is `10` seconds. + - name: processor_start_position + type: select + title: Processor start position + multi: false + required: false + show_user: false + default: earliest + options: + - text: earliest + value: earliest + - text: latest + value: latest + description: >- + (Processor v2 only) Controls from what position in the event hub the processor should start processing messages for all partitions. + + Possible values are `earliest` and `latest`. + + `earliest` starts processing messages from the last checkpoint, or the beginning of the event hub if no checkpoint is available. + + `latest` starts processing messages from the latest event in the event hub and continues to process new events as they arrive. + + Default is `earliest`. + - name: partition_receive_timeout + type: text + title: Partition receive timeout + multi: false + required: false + show_user: false + default: 5s + description: >- + (Processor v2 only) Maximum time to wait before processing the messages received from the event hub. + + The partition consumer waits up to a "receive count" or a "receive timeout", whichever comes first. + + Default is `5` seconds. + - name: partition_receive_count + type: text + title: Partition receive count + multi: false + required: false + show_user: false + default: 100 + description: >- + (Processor v2 only) Maximum number of messages from the event hub to wait for before processing them. + + The partition consumer waits up to a "receive count" or a "receive timeout", whichever comes first. + + Default is `100` messages. - name: transport type: select title: Event Hubs transport protocol @@ -143,6 +202,19 @@ streams: description: > (Processor v2 only) The transport protocol used to connect to Event Hubs. Supported values are AMQP and AMQP-over-WebSockets. Defaults to AMQP. + - name: migrate_checkpoint + type: bool + title: Migrate checkpoint information + multi: false + required: false + show_user: false + default: true + description: >- + (Processor v2 only) Flag to control if the processor should perform the checkpoint information migration from processor v1 to v2 at startup. + + The checkpoint migration converts the checkpoint information from the v1 format to the v2 format, so the processor resumes from where v1 left off instead of reprocessing the event hub retention window. + + Default is `true`. - name: preserve_original_event required: true show_user: true diff --git a/packages/azure_app_service/changelog.yml b/packages/azure_app_service/changelog.yml index 1c1286cf766..5bd2615e210 100644 --- a/packages/azure_app_service/changelog.yml +++ b/packages/azure_app_service/changelog.yml @@ -7,6 +7,9 @@ - description: Add the Event Hub processor version option and switch the default processor version to v2. type: enhancement link: https://github.com/elastic/integrations/pull/20559 + - description: Add the remaining Event Hub processor v2 options (processor update interval, processor start position, partition receive timeout, partition receive count, and checkpoint migration). + type: enhancement + link: https://github.com/elastic/integrations/pull/20559 - version: "1.1.0" changes: - description: Add client secret authentication support for Azure Event Hub with RBAC. diff --git a/packages/azure_app_service/data_stream/app_service_logs/agent/stream/azure-eventhub.yml.hbs b/packages/azure_app_service/data_stream/app_service_logs/agent/stream/azure-eventhub.yml.hbs index b90090afe52..4a64f408d7a 100644 --- a/packages/azure_app_service/data_stream/app_service_logs/agent/stream/azure-eventhub.yml.hbs +++ b/packages/azure_app_service/data_stream/app_service_logs/agent/stream/azure-eventhub.yml.hbs @@ -46,6 +46,19 @@ resource_manager_endpoint: {{resource_manager_endpoint}} {{#if processor_version}} processor_version: {{processor_version}} {{/if}} +migrate_checkpoint: {{migrate_checkpoint}} +{{#if processor_update_interval}} +processor_update_interval: {{processor_update_interval}} +{{/if}} +{{#if processor_start_position}} +processor_start_position: {{processor_start_position}} +{{/if}} +{{#if partition_receive_timeout}} +partition_receive_timeout: {{partition_receive_timeout}} +{{/if}} +{{#if partition_receive_count}} +partition_receive_count: {{partition_receive_count}} +{{/if}} {{#if transport}} transport: {{transport}} {{/if}} diff --git a/packages/azure_app_service/manifest.yml b/packages/azure_app_service/manifest.yml index d7489dec262..53377b85b5b 100644 --- a/packages/azure_app_service/manifest.yml +++ b/packages/azure_app_service/manifest.yml @@ -146,6 +146,65 @@ vars: description: > The processor version that the integration should use. Possible values are v1 (legacy) and v2 (recommended). The v2 event hub processor is recommended for typical use cases. Defaults to v2. + - name: processor_update_interval + type: text + title: Processor update interval + multi: false + required: false + show_user: false + default: 10s + description: >- + (Processor v2 only) How often the processor should attempt to claim partitions. + + Default is `10` seconds. + - name: processor_start_position + type: select + title: Processor start position + multi: false + required: false + show_user: false + default: earliest + options: + - text: earliest + value: earliest + - text: latest + value: latest + description: >- + (Processor v2 only) Controls from what position in the event hub the processor should start processing messages for all partitions. + + Possible values are `earliest` and `latest`. + + `earliest` starts processing messages from the last checkpoint, or the beginning of the event hub if no checkpoint is available. + + `latest` starts processing messages from the latest event in the event hub and continues to process new events as they arrive. + + Default is `earliest`. + - name: partition_receive_timeout + type: text + title: Partition receive timeout + multi: false + required: false + show_user: false + default: 5s + description: >- + (Processor v2 only) Maximum time to wait before processing the messages received from the event hub. + + The partition consumer waits up to a "receive count" or a "receive timeout", whichever comes first. + + Default is `5` seconds. + - name: partition_receive_count + type: text + title: Partition receive count + multi: false + required: false + show_user: false + default: 100 + description: >- + (Processor v2 only) Maximum number of messages from the event hub to wait for before processing them. + + The partition consumer waits up to a "receive count" or a "receive timeout", whichever comes first. + + Default is `100` messages. - name: transport type: select title: Event Hubs transport protocol @@ -161,6 +220,19 @@ vars: description: > (Processor v2 only) The transport protocol used to connect to Event Hubs. Supported values are AMQP and AMQP-over-WebSockets. Defaults to AMQP. + - name: migrate_checkpoint + type: bool + title: Migrate checkpoint information + multi: false + required: false + show_user: false + default: true + description: >- + (Processor v2 only) Flag to control if the processor should perform the checkpoint information migration from processor v1 to v2 at startup. + + The checkpoint migration converts the checkpoint information from the v1 format to the v2 format, so the processor resumes from where v1 left off instead of reprocessing the event hub retention window. + + Default is `true`. icons: - src: /img/app-service-logo.svg title: App Service Logo diff --git a/packages/azure_functions/_dev/build/docs/README.md b/packages/azure_functions/_dev/build/docs/README.md index 43b91fd1143..19fb9f243ee 100644 --- a/packages/azure_functions/_dev/build/docs/README.md +++ b/packages/azure_functions/_dev/build/docs/README.md @@ -154,6 +154,26 @@ _string_ _string_ The Event Hub processor version that the integration should use. Possible values are `v1` (legacy) and `v2` (recommended). Defaults to `v2`. +`processor_update_interval` : +_string_ +(Processor v2 only) How often the processor should attempt to claim partitions. Defaults to `10s`. + +`processor_start_position` : +_string_ +(Processor v2 only) Where the processor starts processing messages for all partitions. `earliest` (default) starts from the last checkpoint, or the beginning of the event hub if no checkpoint is available. `latest` starts from the latest event and continues with new events as they arrive. + +`partition_receive_timeout` : +_string_ +(Processor v2 only) Maximum time to wait before processing the messages received from the event hub. The partition consumer waits up to a "receive count" or a "receive timeout", whichever comes first. Defaults to `5s`. + +`partition_receive_count` : +_int_ +(Processor v2 only) Maximum number of messages from the event hub to wait for before processing them. The partition consumer waits up to a "receive count" or a "receive timeout", whichever comes first. Defaults to `100`. + +`migrate_checkpoint` : +_bool_ +(Processor v2 only) Controls whether the processor migrates checkpoint information from the v1 format to the v2 format at startup, so it resumes from where v1 left off instead of reprocessing the event hub retention window. Defaults to `true`. + `transport` : _string_ (Processor v2 only) The transport protocol to use when connecting to Event Hubs. `amqp` (default) uses ports `5671` and `5672`. `websocket` uses AMQP-over-WebSockets on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. diff --git a/packages/azure_functions/changelog.yml b/packages/azure_functions/changelog.yml index f3597584136..d30988116d3 100644 --- a/packages/azure_functions/changelog.yml +++ b/packages/azure_functions/changelog.yml @@ -7,6 +7,9 @@ - description: Add the Event Hub processor version option and switch the default processor version to v2. type: enhancement link: https://github.com/elastic/integrations/pull/20559 + - description: Add the remaining Event Hub processor v2 options (processor update interval, processor start position, partition receive timeout, partition receive count, and checkpoint migration). + type: enhancement + link: https://github.com/elastic/integrations/pull/20559 - version: "0.15.0" changes: - description: Publish Agentless to Elastic Managed integrations name change. diff --git a/packages/azure_functions/data_stream/functionapplogs/agent/stream/azure-eventhub.yml.hbs b/packages/azure_functions/data_stream/functionapplogs/agent/stream/azure-eventhub.yml.hbs index c39b41969d0..3bb880beb15 100644 --- a/packages/azure_functions/data_stream/functionapplogs/agent/stream/azure-eventhub.yml.hbs +++ b/packages/azure_functions/data_stream/functionapplogs/agent/stream/azure-eventhub.yml.hbs @@ -46,6 +46,19 @@ resource_manager_endpoint: {{resource_manager_endpoint}} {{#if processor_version}} processor_version: {{processor_version}} {{/if}} +migrate_checkpoint: {{migrate_checkpoint}} +{{#if processor_update_interval}} +processor_update_interval: {{processor_update_interval}} +{{/if}} +{{#if processor_start_position}} +processor_start_position: {{processor_start_position}} +{{/if}} +{{#if partition_receive_timeout}} +partition_receive_timeout: {{partition_receive_timeout}} +{{/if}} +{{#if partition_receive_count}} +partition_receive_count: {{partition_receive_count}} +{{/if}} {{#if transport}} transport: {{transport}} {{/if}} diff --git a/packages/azure_functions/data_stream/functionapplogs/manifest.yml b/packages/azure_functions/data_stream/functionapplogs/manifest.yml index 32a2855fec2..11935bdd2d4 100644 --- a/packages/azure_functions/data_stream/functionapplogs/manifest.yml +++ b/packages/azure_functions/data_stream/functionapplogs/manifest.yml @@ -129,6 +129,65 @@ streams: description: > The processor version that the integration should use. Possible values are v1 (legacy) and v2 (recommended). The v2 event hub processor is recommended for typical use cases. Defaults to v2. + - name: processor_update_interval + type: text + title: Processor update interval + multi: false + required: false + show_user: false + default: 10s + description: >- + (Processor v2 only) How often the processor should attempt to claim partitions. + + Default is `10` seconds. + - name: processor_start_position + type: select + title: Processor start position + multi: false + required: false + show_user: false + default: earliest + options: + - text: earliest + value: earliest + - text: latest + value: latest + description: >- + (Processor v2 only) Controls from what position in the event hub the processor should start processing messages for all partitions. + + Possible values are `earliest` and `latest`. + + `earliest` starts processing messages from the last checkpoint, or the beginning of the event hub if no checkpoint is available. + + `latest` starts processing messages from the latest event in the event hub and continues to process new events as they arrive. + + Default is `earliest`. + - name: partition_receive_timeout + type: text + title: Partition receive timeout + multi: false + required: false + show_user: false + default: 5s + description: >- + (Processor v2 only) Maximum time to wait before processing the messages received from the event hub. + + The partition consumer waits up to a "receive count" or a "receive timeout", whichever comes first. + + Default is `5` seconds. + - name: partition_receive_count + type: text + title: Partition receive count + multi: false + required: false + show_user: false + default: 100 + description: >- + (Processor v2 only) Maximum number of messages from the event hub to wait for before processing them. + + The partition consumer waits up to a "receive count" or a "receive timeout", whichever comes first. + + Default is `100` messages. - name: transport type: select title: Event Hubs transport protocol @@ -144,6 +203,19 @@ streams: description: > (Processor v2 only) The transport protocol used to connect to Event Hubs. Supported values are AMQP and AMQP-over-WebSockets. Defaults to AMQP. + - name: migrate_checkpoint + type: bool + title: Migrate checkpoint information + multi: false + required: false + show_user: false + default: true + description: >- + (Processor v2 only) Flag to control if the processor should perform the checkpoint information migration from processor v1 to v2 at startup. + + The checkpoint migration converts the checkpoint information from the v1 format to the v2 format, so the processor resumes from where v1 left off instead of reprocessing the event hub retention window. + + Default is `true`. - name: preserve_original_event required: true show_user: true diff --git a/packages/azure_functions/docs/README.md b/packages/azure_functions/docs/README.md index f56c56aa56c..cf44ea1dbed 100644 --- a/packages/azure_functions/docs/README.md +++ b/packages/azure_functions/docs/README.md @@ -154,6 +154,26 @@ _string_ _string_ The Event Hub processor version that the integration should use. Possible values are `v1` (legacy) and `v2` (recommended). Defaults to `v2`. +`processor_update_interval` : +_string_ +(Processor v2 only) How often the processor should attempt to claim partitions. Defaults to `10s`. + +`processor_start_position` : +_string_ +(Processor v2 only) Where the processor starts processing messages for all partitions. `earliest` (default) starts from the last checkpoint, or the beginning of the event hub if no checkpoint is available. `latest` starts from the latest event and continues with new events as they arrive. + +`partition_receive_timeout` : +_string_ +(Processor v2 only) Maximum time to wait before processing the messages received from the event hub. The partition consumer waits up to a "receive count" or a "receive timeout", whichever comes first. Defaults to `5s`. + +`partition_receive_count` : +_int_ +(Processor v2 only) Maximum number of messages from the event hub to wait for before processing them. The partition consumer waits up to a "receive count" or a "receive timeout", whichever comes first. Defaults to `100`. + +`migrate_checkpoint` : +_bool_ +(Processor v2 only) Controls whether the processor migrates checkpoint information from the v1 format to the v2 format at startup, so it resumes from where v1 left off instead of reprocessing the event hub retention window. Defaults to `true`. + `transport` : _string_ (Processor v2 only) The transport protocol to use when connecting to Event Hubs. `amqp` (default) uses ports `5671` and `5672`. `websocket` uses AMQP-over-WebSockets on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. diff --git a/packages/azure_openai/changelog.yml b/packages/azure_openai/changelog.yml index 68cc73cce39..62c8b12aefb 100644 --- a/packages/azure_openai/changelog.yml +++ b/packages/azure_openai/changelog.yml @@ -7,6 +7,9 @@ - description: Add the Event Hub processor version option and switch the default processor version to v2. type: enhancement link: https://github.com/elastic/integrations/pull/20559 + - description: Add the remaining Event Hub processor v2 options (processor update interval, processor start position, partition receive timeout, partition receive count, and checkpoint migration). + type: enhancement + link: https://github.com/elastic/integrations/pull/20559 - version: "1.14.0" changes: - description: Publish Agentless to Elastic Managed integrations name change. diff --git a/packages/azure_openai/data_stream/logs/agent/stream/azure-eventhub.yml.hbs b/packages/azure_openai/data_stream/logs/agent/stream/azure-eventhub.yml.hbs index 2b5b01dce1a..462cc70f894 100644 --- a/packages/azure_openai/data_stream/logs/agent/stream/azure-eventhub.yml.hbs +++ b/packages/azure_openai/data_stream/logs/agent/stream/azure-eventhub.yml.hbs @@ -46,6 +46,19 @@ resource_manager_endpoint: {{resource_manager_endpoint}} {{#if processor_version}} processor_version: {{processor_version}} {{/if}} +migrate_checkpoint: {{migrate_checkpoint}} +{{#if processor_update_interval}} +processor_update_interval: {{processor_update_interval}} +{{/if}} +{{#if processor_start_position}} +processor_start_position: {{processor_start_position}} +{{/if}} +{{#if partition_receive_timeout}} +partition_receive_timeout: {{partition_receive_timeout}} +{{/if}} +{{#if partition_receive_count}} +partition_receive_count: {{partition_receive_count}} +{{/if}} {{#if transport}} transport: {{transport}} {{/if}} diff --git a/packages/azure_openai/data_stream/logs/manifest.yml b/packages/azure_openai/data_stream/logs/manifest.yml index 0d0ca7f04dc..33675f2c6c4 100644 --- a/packages/azure_openai/data_stream/logs/manifest.yml +++ b/packages/azure_openai/data_stream/logs/manifest.yml @@ -128,6 +128,65 @@ streams: description: > The processor version that the integration should use. Possible values are v1 (legacy) and v2 (recommended). The v2 event hub processor is recommended for typical use cases. Defaults to v2. + - name: processor_update_interval + type: text + title: Processor update interval + multi: false + required: false + show_user: false + default: 10s + description: >- + (Processor v2 only) How often the processor should attempt to claim partitions. + + Default is `10` seconds. + - name: processor_start_position + type: select + title: Processor start position + multi: false + required: false + show_user: false + default: earliest + options: + - text: earliest + value: earliest + - text: latest + value: latest + description: >- + (Processor v2 only) Controls from what position in the event hub the processor should start processing messages for all partitions. + + Possible values are `earliest` and `latest`. + + `earliest` starts processing messages from the last checkpoint, or the beginning of the event hub if no checkpoint is available. + + `latest` starts processing messages from the latest event in the event hub and continues to process new events as they arrive. + + Default is `earliest`. + - name: partition_receive_timeout + type: text + title: Partition receive timeout + multi: false + required: false + show_user: false + default: 5s + description: >- + (Processor v2 only) Maximum time to wait before processing the messages received from the event hub. + + The partition consumer waits up to a "receive count" or a "receive timeout", whichever comes first. + + Default is `5` seconds. + - name: partition_receive_count + type: text + title: Partition receive count + multi: false + required: false + show_user: false + default: 100 + description: >- + (Processor v2 only) Maximum number of messages from the event hub to wait for before processing them. + + The partition consumer waits up to a "receive count" or a "receive timeout", whichever comes first. + + Default is `100` messages. - name: transport type: select title: Event Hubs transport protocol @@ -143,6 +202,19 @@ streams: description: > (Processor v2 only) The transport protocol used to connect to Event Hubs. Supported values are AMQP and AMQP-over-WebSockets. Defaults to AMQP. + - name: migrate_checkpoint + type: bool + title: Migrate checkpoint information + multi: false + required: false + show_user: false + default: true + description: >- + (Processor v2 only) Flag to control if the processor should perform the checkpoint information migration from processor v1 to v2 at startup. + + The checkpoint migration converts the checkpoint information from the v1 format to the v2 format, so the processor resumes from where v1 left off instead of reprocessing the event hub retention window. + + Default is `true`. - name: preserve_original_event required: true show_user: true From bf596021a14d2575800c21d616a30fd160f11540 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Fri, 7 Aug 2026 07:49:23 +0200 Subject: [PATCH 5/5] [Azure] Document optional proxy setup for AMQP-over-WebSockets transport --- .../azure_ai_foundry/_dev/build/docs/README.md | 15 +++++++++++++-- packages/azure_ai_foundry/docs/README.md | 15 +++++++++++++-- .../azure_app_service/_dev/build/docs/README.md | 15 +++++++++++++-- packages/azure_app_service/docs/README.md | 15 +++++++++++++-- .../azure_functions/_dev/build/docs/README.md | 15 +++++++++++++-- packages/azure_functions/docs/README.md | 15 +++++++++++++-- packages/azure_logs/_dev/build/docs/README.md | 9 +++++++-- packages/azure_logs/docs/README.md | 9 +++++++-- packages/azure_openai/_dev/build/docs/README.md | 15 +++++++++++++-- packages/azure_openai/docs/README.md | 15 +++++++++++++-- 10 files changed, 118 insertions(+), 20 deletions(-) diff --git a/packages/azure_ai_foundry/_dev/build/docs/README.md b/packages/azure_ai_foundry/_dev/build/docs/README.md index 9a398526511..1d9b36a7bd3 100644 --- a/packages/azure_ai_foundry/_dev/build/docs/README.md +++ b/packages/azure_ai_foundry/_dev/build/docs/README.md @@ -36,12 +36,23 @@ Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for m **Authentication (Event Hub):** The Event Hub input supports two authentication methods: **connection string** (default) and **client secret** (Microsoft Entra ID). For setup steps, required RBAC roles (Azure Event Hubs Data Receiver, Storage Blob Data Contributor), and configuration options, see the [Azure Logs integration](https://docs.elastic.co/integrations/azure) or [Filebeat azure-eventhub input](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-eventhub.html) documentation. -#### AMQP-over-WebSockets and proxy support +#### AMQP-over-WebSockets -By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. +By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443`. This option requires the Event Hub processor v2 (**Processor version** set to `v2`, which is the default) and Elastic Agent 9.2.4 or later. +#### Proxy support + +Proxy support is optional and requires **Event Hubs transport protocol** set to **AMQP-over-WebSockets**. + +To enable it: + +1. In the advanced options, set **Event Hubs transport protocol** to **AMQP-over-WebSockets**. +2. Define the `HTTPS_PROXY` environment variable for the Elastic Agent process, for example `HTTPS_PROXY=http://proxy.example.com:8080`. Elastic Agent routes both Event Hubs and Storage Account traffic through the proxy. + +This requires processor v2 and Elastic Agent 9.2.4 or later. + #### Native logging The Microsoft Foundry provides native logging and monitoring to track the telemetry of the service. The `Audit` and `RequestResponse` log categories come under the native logging. However, the default logging doesn't log the inputs and outputs of the service. This is useful to ensure that the services operates as expected. diff --git a/packages/azure_ai_foundry/docs/README.md b/packages/azure_ai_foundry/docs/README.md index 3f065e9e83a..9706fb12841 100644 --- a/packages/azure_ai_foundry/docs/README.md +++ b/packages/azure_ai_foundry/docs/README.md @@ -36,12 +36,23 @@ Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for m **Authentication (Event Hub):** The Event Hub input supports two authentication methods: **connection string** (default) and **client secret** (Microsoft Entra ID). For setup steps, required RBAC roles (Azure Event Hubs Data Receiver, Storage Blob Data Contributor), and configuration options, see the [Azure Logs integration](https://docs.elastic.co/integrations/azure) or [Filebeat azure-eventhub input](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-eventhub.html) documentation. -#### AMQP-over-WebSockets and proxy support +#### AMQP-over-WebSockets -By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. +By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443`. This option requires the Event Hub processor v2 (**Processor version** set to `v2`, which is the default) and Elastic Agent 9.2.4 or later. +#### Proxy support + +Proxy support is optional and requires **Event Hubs transport protocol** set to **AMQP-over-WebSockets**. + +To enable it: + +1. In the advanced options, set **Event Hubs transport protocol** to **AMQP-over-WebSockets**. +2. Define the `HTTPS_PROXY` environment variable for the Elastic Agent process, for example `HTTPS_PROXY=http://proxy.example.com:8080`. Elastic Agent routes both Event Hubs and Storage Account traffic through the proxy. + +This requires processor v2 and Elastic Agent 9.2.4 or later. + #### Native logging The Microsoft Foundry provides native logging and monitoring to track the telemetry of the service. The `Audit` and `RequestResponse` log categories come under the native logging. However, the default logging doesn't log the inputs and outputs of the service. This is useful to ensure that the services operates as expected. diff --git a/packages/azure_app_service/_dev/build/docs/README.md b/packages/azure_app_service/_dev/build/docs/README.md index 4fefd1c64f0..a31ce8e6445 100644 --- a/packages/azure_app_service/_dev/build/docs/README.md +++ b/packages/azure_app_service/_dev/build/docs/README.md @@ -23,12 +23,23 @@ Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for m **Authentication (Event Hub):** The Event Hub input supports two authentication methods: **connection string** (default) and **client secret** (Microsoft Entra ID). For setup steps, required RBAC roles (Azure Event Hubs Data Receiver, Storage Blob Data Contributor), and configuration options, see the [Azure Logs integration](https://docs.elastic.co/integrations/azure) or [Filebeat azure-eventhub input](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-eventhub.html) documentation. -### AMQP-over-WebSockets and proxy support +### AMQP-over-WebSockets -By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. +By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443`. This option requires the Event Hub processor v2 (**Processor version** set to `v2`, which is the default) and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. +### Proxy support + +Proxy support is optional and requires **Event Hubs transport protocol** set to **AMQP-over-WebSockets**. + +To enable it: + +1. In the advanced options, set **Event Hubs transport protocol** to **AMQP-over-WebSockets**. +2. Define the `HTTPS_PROXY` environment variable for the Elastic Agent process, for example `HTTPS_PROXY=http://proxy.example.com:8080`. Elastic Agent routes both Event Hubs and Storage Account traffic through the proxy. + +This requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. + ## App Service Logs Collects different types of logs from Azure App Service via Event Hub. diff --git a/packages/azure_app_service/docs/README.md b/packages/azure_app_service/docs/README.md index 84cf75a934d..38f3c90958f 100644 --- a/packages/azure_app_service/docs/README.md +++ b/packages/azure_app_service/docs/README.md @@ -23,12 +23,23 @@ Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for m **Authentication (Event Hub):** The Event Hub input supports two authentication methods: **connection string** (default) and **client secret** (Microsoft Entra ID). For setup steps, required RBAC roles (Azure Event Hubs Data Receiver, Storage Blob Data Contributor), and configuration options, see the [Azure Logs integration](https://docs.elastic.co/integrations/azure) or [Filebeat azure-eventhub input](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-eventhub.html) documentation. -### AMQP-over-WebSockets and proxy support +### AMQP-over-WebSockets -By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. +By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443`. This option requires the Event Hub processor v2 (**Processor version** set to `v2`, which is the default) and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. +### Proxy support + +Proxy support is optional and requires **Event Hubs transport protocol** set to **AMQP-over-WebSockets**. + +To enable it: + +1. In the advanced options, set **Event Hubs transport protocol** to **AMQP-over-WebSockets**. +2. Define the `HTTPS_PROXY` environment variable for the Elastic Agent process, for example `HTTPS_PROXY=http://proxy.example.com:8080`. Elastic Agent routes both Event Hubs and Storage Account traffic through the proxy. + +This requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. + ## App Service Logs Collects different types of logs from Azure App Service via Event Hub. diff --git a/packages/azure_functions/_dev/build/docs/README.md b/packages/azure_functions/_dev/build/docs/README.md index 19fb9f243ee..3872ce79b5e 100644 --- a/packages/azure_functions/_dev/build/docs/README.md +++ b/packages/azure_functions/_dev/build/docs/README.md @@ -96,12 +96,23 @@ Instead of a connection string, you can authenticate using a Microsoft Entra ID 4. **Configure the integration** Set **Authentication type** to **Client Secret**. Provide **Tenant ID**, **Client ID**, **Client Secret**, and the fully qualified **Event Hub namespace** (for example `yournamespace.servicebus.windows.net`). Use the same Storage Account and container as for connection string authentication; the integration will use the client secret to access both Event Hubs and Storage. -#### AMQP-over-WebSockets and proxy support +#### AMQP-over-WebSockets -By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. +By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443`. This option requires the Event Hub processor v2 (**Processor version** set to `v2`, which is the default) and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. +#### Proxy support + +Proxy support is optional and requires **Event Hubs transport protocol** set to **AMQP-over-WebSockets**. + +To enable it: + +1. In the advanced options, set **Event Hubs transport protocol** to **AMQP-over-WebSockets**. +2. Define the `HTTPS_PROXY` environment variable for the Elastic Agent process, for example `HTTPS_PROXY=http://proxy.example.com:8080`. Elastic Agent routes both Event Hubs and Storage Account traffic through the proxy. + +This requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. + #### Configuration options `auth_type` : diff --git a/packages/azure_functions/docs/README.md b/packages/azure_functions/docs/README.md index cf44ea1dbed..c54d8ded737 100644 --- a/packages/azure_functions/docs/README.md +++ b/packages/azure_functions/docs/README.md @@ -96,12 +96,23 @@ Instead of a connection string, you can authenticate using a Microsoft Entra ID 4. **Configure the integration** Set **Authentication type** to **Client Secret**. Provide **Tenant ID**, **Client ID**, **Client Secret**, and the fully qualified **Event Hub namespace** (for example `yournamespace.servicebus.windows.net`). Use the same Storage Account and container as for connection string authentication; the integration will use the client secret to access both Event Hubs and Storage. -#### AMQP-over-WebSockets and proxy support +#### AMQP-over-WebSockets -By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. +By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443`. This option requires the Event Hub processor v2 (**Processor version** set to `v2`, which is the default) and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. +#### Proxy support + +Proxy support is optional and requires **Event Hubs transport protocol** set to **AMQP-over-WebSockets**. + +To enable it: + +1. In the advanced options, set **Event Hubs transport protocol** to **AMQP-over-WebSockets**. +2. Define the `HTTPS_PROXY` environment variable for the Elastic Agent process, for example `HTTPS_PROXY=http://proxy.example.com:8080`. Elastic Agent routes both Event Hubs and Storage Account traffic through the proxy. + +This requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. + #### Configuration options `auth_type` : diff --git a/packages/azure_logs/_dev/build/docs/README.md b/packages/azure_logs/_dev/build/docs/README.md index bec48226380..30207557019 100644 --- a/packages/azure_logs/_dev/build/docs/README.md +++ b/packages/azure_logs/_dev/build/docs/README.md @@ -391,9 +391,14 @@ Optionally, you can restrict the traffic to the following domain names: #### Proxy support -When using AMQP-over-WebSockets, both Event Hubs and Storage Account traffic use HTTPS on port `443`. Set the `HTTPS_PROXY` environment variable to route this traffic through a proxy. +Proxy support is optional and requires **Event Hubs transport protocol** set to **AMQP-over-WebSockets**. -Proxy support requires **Event Hubs transport protocol** set to **AMQP-over-WebSockets**, processor v2, and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. +To enable it: + +1. In the advanced options, set **Event Hubs transport protocol** to **AMQP-over-WebSockets**. +2. Define the `HTTPS_PROXY` environment variable for the Elastic Agent process, for example `HTTPS_PROXY=http://proxy.example.com:8080`. Elastic Agent routes both Event Hubs and Storage Account traffic through the proxy. + +This requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. ## Settings diff --git a/packages/azure_logs/docs/README.md b/packages/azure_logs/docs/README.md index bec48226380..30207557019 100644 --- a/packages/azure_logs/docs/README.md +++ b/packages/azure_logs/docs/README.md @@ -391,9 +391,14 @@ Optionally, you can restrict the traffic to the following domain names: #### Proxy support -When using AMQP-over-WebSockets, both Event Hubs and Storage Account traffic use HTTPS on port `443`. Set the `HTTPS_PROXY` environment variable to route this traffic through a proxy. +Proxy support is optional and requires **Event Hubs transport protocol** set to **AMQP-over-WebSockets**. -Proxy support requires **Event Hubs transport protocol** set to **AMQP-over-WebSockets**, processor v2, and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. +To enable it: + +1. In the advanced options, set **Event Hubs transport protocol** to **AMQP-over-WebSockets**. +2. Define the `HTTPS_PROXY` environment variable for the Elastic Agent process, for example `HTTPS_PROXY=http://proxy.example.com:8080`. Elastic Agent routes both Event Hubs and Storage Account traffic through the proxy. + +This requires processor v2 and Elastic Agent 8.19.10, 9.1.10, 9.2.4, or later. ## Settings diff --git a/packages/azure_openai/_dev/build/docs/README.md b/packages/azure_openai/_dev/build/docs/README.md index 6b50d372a2e..a799c3e65d9 100644 --- a/packages/azure_openai/_dev/build/docs/README.md +++ b/packages/azure_openai/_dev/build/docs/README.md @@ -41,12 +41,23 @@ Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for m **Authentication (Event Hub):** The Event Hub input supports two authentication methods: **connection string** (default) and **client secret** (Microsoft Entra ID). For setup steps, required RBAC roles (Azure Event Hubs Data Receiver, Storage Blob Data Contributor), and configuration options, see the [Azure Logs integration](https://docs.elastic.co/integrations/azure) or [Filebeat azure-eventhub input](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-eventhub.html) documentation. -#### AMQP-over-WebSockets and proxy support +#### AMQP-over-WebSockets -By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. +By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443`. This option requires the Event Hub processor v2 (**Processor version** set to `v2`, which is the default) and Elastic Agent 8.19.10, 9.2.4, or later. +#### Proxy support + +Proxy support is optional and requires **Event Hubs transport protocol** set to **AMQP-over-WebSockets**. + +To enable it: + +1. In the advanced options, set **Event Hubs transport protocol** to **AMQP-over-WebSockets**. +2. Define the `HTTPS_PROXY` environment variable for the Elastic Agent process, for example `HTTPS_PROXY=http://proxy.example.com:8080`. Elastic Agent routes both Event Hubs and Storage Account traffic through the proxy. + +This requires processor v2 and Elastic Agent 8.19.10, 9.2.4, or later. + #### Default Logging The Azure OpenAI provides native logging and monitoring to track the telemetry of the service. The Audit and RequestResponse log categories come under the native logging. However, the default logging doesn't log the inputs and outputs of the service. This is useful to ensure that the services operates as expected. diff --git a/packages/azure_openai/docs/README.md b/packages/azure_openai/docs/README.md index d0bdf019cc0..cfe7884817f 100644 --- a/packages/azure_openai/docs/README.md +++ b/packages/azure_openai/docs/README.md @@ -41,12 +41,23 @@ Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for m **Authentication (Event Hub):** The Event Hub input supports two authentication methods: **connection string** (default) and **client secret** (Microsoft Entra ID). For setup steps, required RBAC roles (Azure Event Hubs Data Receiver, Storage Blob Data Contributor), and configuration options, see the [Azure Logs integration](https://docs.elastic.co/integrations/azure) or [Filebeat azure-eventhub input](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-eventhub.html) documentation. -#### AMQP-over-WebSockets and proxy support +#### AMQP-over-WebSockets -By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443` and enables proxy support through the `HTTPS_PROXY` environment variable. +By default, the integration uses AMQP ports `5671` and `5672` to communicate with Event Hubs. If these ports are blocked, set **Event Hubs transport protocol** to **AMQP-over-WebSockets** in the advanced options. This tunnels AMQP over HTTPS on port `443`. This option requires the Event Hub processor v2 (**Processor version** set to `v2`, which is the default) and Elastic Agent 8.19.10, 9.2.4, or later. +#### Proxy support + +Proxy support is optional and requires **Event Hubs transport protocol** set to **AMQP-over-WebSockets**. + +To enable it: + +1. In the advanced options, set **Event Hubs transport protocol** to **AMQP-over-WebSockets**. +2. Define the `HTTPS_PROXY` environment variable for the Elastic Agent process, for example `HTTPS_PROXY=http://proxy.example.com:8080`. Elastic Agent routes both Event Hubs and Storage Account traffic through the proxy. + +This requires processor v2 and Elastic Agent 8.19.10, 9.2.4, or later. + #### Default Logging The Azure OpenAI provides native logging and monitoring to track the telemetry of the service. The Audit and RequestResponse log categories come under the native logging. However, the default logging doesn't log the inputs and outputs of the service. This is useful to ensure that the services operates as expected.