diff --git a/manual/pom.xml b/manual/pom.xml index 82c7b48583..b1dc256952 100644 --- a/manual/pom.xml +++ b/manual/pom.xml @@ -33,8 +33,9 @@ target/generated-docs/html/latest target/generated-docs/pdf/latest ${project.version} - - ${env.GRAPHVIZ_DOT} + + dot @@ -72,9 +73,6 @@ asciidoctor-diagram - - ${graphviz.dot.path} - index.adoc ${doc.source} ${doc.output.html} @@ -89,6 +87,8 @@ highlightjs ${project.version} ${project.version} + ${graphviz.dot.path} + ${project.basedir}/src/main/asciidoc/plantuml/unomi-theme.puml @@ -122,6 +122,8 @@ _ true + ${graphviz.dot.path} + ${project.basedir}/src/main/asciidoc/plantuml/unomi-theme.puml @@ -207,6 +209,18 @@ + + + graphviz-from-env + + + env.GRAPHVIZ_DOT + + + + ${env.GRAPHVIZ_DOT} + + generate-documentation-bundle diff --git a/manual/src/main/asciidoc/5-min-quickstart.adoc b/manual/src/main/asciidoc/5-min-quickstart.adoc index e1aacb07d3..c42a9404a7 100644 --- a/manual/src/main/asciidoc/5-min-quickstart.adoc +++ b/manual/src/main/asciidoc/5-min-quickstart.adoc @@ -15,9 +15,9 @@ [#_five_minutes_quickstart] === Quick start with Docker -Begin by creating a `docker-compose.yml` file. You can choose between ElasticSearch or OpenSearch: +Begin by creating a `docker-compose.yml` file. You can choose between Elasticsearch or OpenSearch: -==== Option 1: Using ElasticSearch +==== Option 1: Using Elasticsearch [source,yaml] ---- @@ -52,14 +52,14 @@ services: version: '3.8' services: opensearch-node1: - image: opensearchproject/opensearch:3 + image: opensearchproject/opensearch:3.7.0 environment: - cluster.name=opensearch-cluster - node.name=opensearch-node1 - discovery.type=single-node - bootstrap.memory_lock=true - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" - - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-admin} + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-MyStrongPassw0rd!} ulimits: memlock: soft: -1 @@ -79,7 +79,10 @@ services: - UNOMI_DISTRIBUTION=unomi-distribution-opensearch - UNOMI_OPENSEARCH_ADDRESSES=opensearch-node1:9200 - UNOMI_OPENSEARCH_USERNAME=admin - - UNOMI_OPENSEARCH_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-admin} + - UNOMI_OPENSEARCH_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-MyStrongPassw0rd!} + # Package default sslEnable is true; trust-all matches typical local OpenSearch Docker TLS + - UNOMI_OPENSEARCH_SSL_ENABLE=true + - UNOMI_OPENSEARCH_SSL_TRUST_ALL_CERTIFICATES=true - UNOMI_HEALTHCHECK_PROVIDERS=cluster,opensearch,unomi,persistence ports: - 8181:8181 @@ -94,9 +97,9 @@ volumes: From the same folder, start the environment using `docker-compose up` and wait for the startup to complete. -==== After startup (ElasticSearch Docker path) +==== After startup (Elasticsearch Docker path) -Once Unomi is running, create a tenant and save the API keys from the response: +Once Unomi is running, create a tenant, then **regenerate** API keys and save the `plainTextKey` values from those responses (tenant create only returns masked keys): [source,bash] ---- @@ -110,6 +113,9 @@ curl -X POST http://localhost:8181/cxs/tenants \ "description": "Default tenant for quick start" } }' + +curl -X POST "http://localhost:8181/cxs/tenants/default/apikeys?type=PUBLIC" --user karaf:karaf +curl -X POST "http://localhost:8181/cxs/tenants/default/apikeys?type=PRIVATE" --user karaf:karaf ---- Use the public API key in `X-Unomi-Api-Key` for `/cxs/context.json` requests. See <<_multitenancy,Multi-tenancy>>. @@ -118,26 +124,26 @@ Try accessing https://localhost:9443/cxs/cluster with username/password: karaf/k === Quick Start manually -==== Option 1: Using ElasticSearch +==== Option 1: Using Elasticsearch 1) Install JDK 17 and make sure you set the JAVA_HOME variable (see our <<_jdk_compatibility,Getting Started>> guide for more information on JDK compatibility) -2) Download ElasticSearch here : https://www.elastic.co/downloads/past-releases/elasticsearch-9-4-3 (please *make sure* you use the proper version : 9.4.3) +2) Download Elasticsearch here : https://www.elastic.co/downloads/past-releases/elasticsearch-9-4-3 (please *make sure* you use the proper version : 9.4.3) 3) Uncompress it and change the `config/elasticsearch.yml` to include the following config : [source,yaml] ---- -cluster.name: contextElasticSearch +cluster.name: contextElasticsearch ---- -4) Launch ElasticSearch using : `bin/elasticsearch` +4) Launch Elasticsearch using : `bin/elasticsearch` ==== Option 2: Using OpenSearch 1) Install JDK 17 as described above -2) Download OpenSearch here: https://opensearch.org/downloads.html (please *make sure* you use version 3.x) +2) Download OpenSearch here: https://opensearch.org/downloads.html (please *make sure* you use version **3.7.0**, matching `opensearch.version` in the Unomi root POM) 3) Uncompress it and change the `config/opensearch.yml` to include the following config: @@ -169,7 +175,7 @@ which determines which set of features and bundles are installed and started. A 9) Try accessing https://localhost:9443/cxs/cluster with username/password: `karaf/karaf` . You might get a certificate warning in your browser, just accept it despite the warning it is safe. -10) Create a tenant that will own all your data: +10) Create a tenant that will own all your data, then regenerate keys and store `plainTextKey`: [source,bash] ---- @@ -183,9 +189,12 @@ curl -X POST http://localhost:8181/cxs/tenants \ "description": "Default tenant for quick start" } }' + +curl -X POST "http://localhost:8181/cxs/tenants/default/apikeys?type=PUBLIC" --user karaf:karaf +curl -X POST "http://localhost:8181/cxs/tenants/default/apikeys?type=PRIVATE" --user karaf:karaf ---- -Save the API keys from the response - you'll need them for API calls. +Save the `plainTextKey` values from the key-creation responses — you'll need them for API calls. 11) Request your first context: @@ -216,6 +225,7 @@ Next steps: - Trying our integration <<_samples,samples page>> Note: When using OpenSearch, make sure to: -- Set up proper SSL certificates or disable SSL verification for development -- Configure the admin password via OPENSEARCH_INITIAL_ADMIN_PASSWORD -- Enable SSL in Unomi configuration if using secure connections +- Align SSL with the cluster (`UNOMI_OPENSEARCH_SSL_ENABLE` / `org.apache.unomi.opensearch.sslEnable`; package default is `true`) +- For local Docker TLS, set `UNOMI_OPENSEARCH_SSL_TRUST_ALL_CERTIFICATES=true` or install a proper trust store +- Configure a strong admin password via `OPENSEARCH_INITIAL_ADMIN_PASSWORD` (OpenSearch rejects weak passwords such as `admin`) +- Select the OpenSearch distribution (`UNOMI_DISTRIBUTION=unomi-distribution-opensearch` or `unomi:setup -d=unomi-distribution-opensearch`) diff --git a/manual/src/main/asciidoc/architecture.adoc b/manual/src/main/asciidoc/architecture.adoc index 383809d11c..2b93ee8199 100644 --- a/manual/src/main/asciidoc/architecture.adoc +++ b/manual/src/main/asciidoc/architecture.adoc @@ -14,7 +14,7 @@ === High-Level Architecture -NOTE: Unomi 3.1 adds <<_multitenancy,multi-tenancy>>, <<_scheduler,task scheduling>>, and <<_clustering,persistence-based clustering>>. The diagram below focuses on core request processing. +NOTE: Unomi 3.1 adds <<_multitenancy,multi-tenancy>> and <<_scheduler,task scheduling>> on top of the 3.0 platform (including <<_clustering,persistence-based clustering>>). The diagram below focuses on core request processing. [plantuml] ---- @@ -61,11 +61,6 @@ Rel(karaf, unomi, "Hosts") ---- @startuml skinparam componentStyle uml2 -skinparam component { - BackgroundColor<> LightBlue - BackgroundColor<> LightGreen - BackgroundColor<> LightYellow -} package "Core Services" { [Profile Service] <> @@ -141,11 +136,6 @@ end note ---- @startuml skinparam componentStyle uml2 -skinparam component { - BackgroundColor<> LightBlue - BackgroundColor<> LightGreen - BackgroundColor<> LightYellow -} package "Condition Evaluation" { [ConditionDispatcher] <> @@ -213,9 +203,6 @@ end note [plantuml] ---- @startuml -skinparam activityBackgroundColor LightBlue -skinparam activityBorderColor DarkBlue -skinparam arrowColor DarkBlue |Client| start diff --git a/manual/src/main/asciidoc/building-and-deploying.adoc b/manual/src/main/asciidoc/building-and-deploying.adoc index ad9987d177..dc15533260 100644 --- a/manual/src/main/asciidoc/building-and-deploying.adoc +++ b/manual/src/main/asciidoc/building-and-deploying.adoc @@ -242,7 +242,7 @@ To merge the branch into master. Apache Unomi 3.x does not embed a search engine. Install Elasticsearch 9.x or OpenSearch 3.x as a standalone service. See <<_migrate_from_2_x_to_3_0,Migrate from 2.x to 3.0>> for version requirements. -===== Option 1: Using ElasticSearch +===== Option 1: Using Elasticsearch 1. Download Elasticsearch 9.4.3 from: https://www.elastic.co/downloads/past-releases/elasticsearch-9-4-3[https://www.elastic.co/downloads/past-releases/elasticsearch-9-4-3] @@ -252,7 +252,7 @@ Apache Unomi 3.x does not embed a search engine. Install Elasticsearch 9.x or Op [source,yaml] ---- -cluster.name: contextElasticSearch +cluster.name: contextElasticsearch ---- + 4. Launch the server using: @@ -274,14 +274,14 @@ The recommended way to run OpenSearch is using Docker Compose: version: '3.8' services: opensearch-node1: - image: opensearchproject/opensearch:3 + image: opensearchproject/opensearch:3.7.0 environment: - cluster.name=opensearch-cluster - node.name=opensearch-node1 - discovery.type=single-node - bootstrap.memory_lock=true - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" - - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-admin} + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-MyStrongPassw0rd!} ulimits: memlock: soft: -1 @@ -324,7 +324,7 @@ After your search engine is running, you can start Unomi using the appropriate c [source] ---- -# For ElasticSearch +# For Elasticsearch unomi:start elasticsearch # For OpenSearch @@ -470,7 +470,7 @@ which will output something like this : [source] ---- Matching Java Virtual Machines (3): - 11.0.5, x86_64: "OpenJDK 11.0.5" /Library/Java/JavaVirtualMachines/openjdk-11.jdk/Contents/Home + 17.0.x, x86_64: "OpenJDK 17" /Library/Java/JavaVirtualMachines/openjdk-17.jdk/Contents/Home 1.8.0_181, x86_64: "Java SE 8" /Library/Java/JavaVirtualMachines/jdk1.8.0_181.jdk/Contents/Home 1.7.0_80, x86_64: "Java SE 7" /Library/Java/JavaVirtualMachines/jdk1.7.0_80.jdk/Contents/Home @@ -507,7 +507,7 @@ integration tests at least once before using the server to make sure that everyt to use these tests is to run them from a continuous integration server such as Jenkins, Apache Gump, Atlassian Bamboo or others. -Note : the integration tests require a JDK 11 or more recent ! +Note : the integration tests require a JDK **17** or more recent ! To run the tests simply activate the following profile: @@ -518,7 +518,7 @@ mvn -P integration-tests clean install ===== Selecting the Search Engine for Integration Tests -By default, integration tests target ElasticSearch. To run them against OpenSearch, you can use either: +By default, integration tests target Elasticsearch. To run them against OpenSearch, you can use either: * **Using the build script** (recommended): + @@ -557,6 +557,7 @@ A default test page is provided at the following URL: http://localhost:8181/index.html ---- -This test page will trigger the loading of the /cxs/context.js script, which will try to retrieving the user context -or create a new one if it doesn't exist yet. It also contains an experimental integration with Facebook Login, but it -doesn't yet save the context back to the context server. +This test page loads `/cxs/context.js`, which retrieves or creates a user context. +On Unomi 3.1, public context endpoints require a tenant public API key (`X-Unomi-Api-Key`) unless <<_v2_compatibility_mode,V2 compatibility mode>> is enabled. +Create a tenant and regenerate keys first (see <<_five_minutes_quickstart,5-minute quickstart>> or <<_multitenancy,Multi-tenancy>>). +The page also contains an experimental Facebook Login integration that does not yet save the context back to the server. diff --git a/manual/src/main/asciidoc/condition-evaluation.adoc b/manual/src/main/asciidoc/condition-evaluation.adoc index 2171d911c4..f9fcc4a92b 100644 --- a/manual/src/main/asciidoc/condition-evaluation.adoc +++ b/manual/src/main/asciidoc/condition-evaluation.adoc @@ -26,11 +26,6 @@ The condition evaluation system in Apache Unomi provides flexible and efficient ---- @startuml skinparam componentStyle uml2 -skinparam component { - BackgroundColor<> LightBlue - BackgroundColor<> LightGreen - BackgroundColor<> LightYellow -} package "Condition Evaluation" { [ConditionDispatcher] <> @@ -41,12 +36,12 @@ package "Condition Evaluation" { package "Query Building" { interface "QueryBuilder" as QB - [ElasticSearchQueryBuilder] <> + [ElasticsearchQueryBuilder] <> [OpenSearchQueryBuilder] <> } package "Storage" { - [ElasticSearch] <> + [Elasticsearch] <> [OpenSearch] <> } @@ -56,10 +51,10 @@ package "Storage" { QB <|.. [ConditionQueryBuilder] QB <|.. [ConditionQueryBuilder] -[ConditionQueryBuilder] --> [ElasticSearchQueryBuilder] +[ConditionQueryBuilder] --> [ElasticsearchQueryBuilder] [ConditionQueryBuilder] --> [OpenSearchQueryBuilder] -[ElasticSearchQueryBuilder] --> [ElasticSearch] +[ElasticsearchQueryBuilder] --> [Elasticsearch] [OpenSearchQueryBuilder] --> [OpenSearch] note right of [ConditionDispatcher] @@ -111,7 +106,7 @@ end note Query builders provide storage-specific implementations: -1. *ElasticSearch Implementation* +1. *Elasticsearch Implementation* - Optimized for ES query syntax - Handles ES-specific mappings - Supports ES versioning diff --git a/manual/src/main/asciidoc/configuration.adoc b/manual/src/main/asciidoc/configuration.adoc index c1318b2994..b76b2abb2e 100644 --- a/manual/src/main/asciidoc/configuration.adoc +++ b/manual/src/main/asciidoc/configuration.adoc @@ -72,12 +72,12 @@ If you need to specify a search engine configuration that is different than the it is recommended to do this BEFORE you start the server for the first time, or you will lose all the data you have stored previously. -Apache Unomi supports both ElasticSearch and OpenSearch as search engine backends. Here are the configuration properties for each: +Apache Unomi supports both Elasticsearch and OpenSearch as search engine backends. Here are the configuration properties for each: -For ElasticSearch: +For Elasticsearch: [source] ---- -org.apache.unomi.elasticsearch.cluster.name=contextElasticSearch +org.apache.unomi.elasticsearch.cluster.name=contextElasticsearch # The elasticsearch.addresses may be a comma separated list of host names and ports such as # hostA:9200,hostB:9200 # Note: the port number must be repeated for each host. @@ -108,11 +108,11 @@ org.apache.unomi.opensearch.sslTrustAllCertificates=true To select which search engine to use, you can: 1. Use the appropriate configuration properties above 2. When building from source, use the appropriate Maven profile: - * For ElasticSearch (default): no special profile needed + * For Elasticsearch (default): no special profile needed * For OpenSearch: add `-Duse.opensearch=true` to your Maven command (this will only impact the integration tests if they are activated using the -Pintegration-tests profile) 3. When using Docker: - * For ElasticSearch: use `UNOMI_DISTRIBUTION=unomi-distribution-elasticsearch` + * For Elasticsearch: use `UNOMI_DISTRIBUTION=unomi-distribution-elasticsearch` * For OpenSearch: use `UNOMI_DISTRIBUTION=unomi-distribution-opensearch` * For custom configurations: use `UNOMI_DISTRIBUTION=your-custom-distribution-name` @@ -125,9 +125,9 @@ Note: When using OpenSearch 3.x: === Refresh Policy Configuration -Apache Unomi provides the ability to configure refresh policies for different item types in both ElasticSearch and OpenSearch. This feature allows you to optimize when changes to documents are made visible to search. +Apache Unomi provides the ability to configure refresh policies for different item types in both Elasticsearch and OpenSearch. This feature allows you to optimize when changes to documents are made visible to search. -For ElasticSearch: +For Elasticsearch: [source] ---- # refresh policy per item type in Json format. @@ -230,6 +230,7 @@ http://dev.maxmind.com/geoip/geoip2/geolite2/[http://dev.maxmind.com/geoip/geoip Simply download the GeoLite2-City.mmdb file into the "etc" directory. +[#_installing_geonames_database] === Installing Geonames database Apache Unomi includes a geocoding service based on the geonames database ( http://www.geonames.org/[http://www.geonames.org/] ). It can be @@ -296,18 +297,21 @@ curl -X POST "http://localhost:8181/cxs/tenants" \ } }' -# Response (HTTP 201 Created): +# Response (HTTP 201 Created) — apiKeys are masked only; regenerate to obtain plaintext: { - "requestedId": "mytenant", - "name": "My Company", - "description": "My Company tenant", + "itemId": "mytenant", + "itemType": "tenant", + "status": "ACTIVE", + "apiKeys": [ + {"maskedKey": "unomi_v1_****ab12", "keyType": "PUBLIC", "revoked": false}, + {"maskedKey": "unomi_v1_****cd34", "keyType": "PRIVATE", "revoked": false} + ], "properties": { + "name": "My Company", + "description": "My Company tenant", "address": "123 Main St", "country": "USA" }, - "itemType": "tenant", - "version": 1, - "status": "ACTIVE", "creationDate": "2024-03-14T10:30:00Z", "lastModificationDate": "2024-03-14T10:30:00Z" } @@ -350,38 +354,46 @@ Each tenant has two types of API keys: * Public API Key: Used for client-side operations and public endpoints * Private API Key: Used for secure operations and administrative tasks -The API keys are automatically generated when creating a tenant. You can view them using: +The API keys are generated when creating a tenant, but the tenant resource only returns **masked** key metadata. To obtain the one-time plaintext value, regenerate keys (this replaces any existing key of that type): [source,bash] ---- -# Using Karaf shell (requires admin access) +# Using Karaf shell (requires admin access) — metadata only unomi:crud read tenant -i mytenant -# Output example: -Tenant Details: -ID: mytenant -Name: My Company -Description: My Company tenant -Status: ACTIVE -Creation Date: 2024-03-14T10:30:00Z -Last Modified: 2024-03-14T10:30:00Z -Public API Key: 8f7d9a2c-5e4b-3f1a-9b8c-7d6e5f4a3b2c -Private API Key: 1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6 +# Obtain plaintext (store immediately; it is not persisted) +curl -X POST "http://localhost:8181/cxs/tenants/mytenant/apikeys?type=PUBLIC" \ + -u karaf:karaf +curl -X POST "http://localhost:8181/cxs/tenants/mytenant/apikeys?type=PRIVATE" \ + -u karaf:karaf + +# Response example: +{ + "apiKey": { + "maskedKey": "unomi_v1_****ab12", + "keyType": "PUBLIC", + "revoked": false + }, + "plainTextKey": "unomi_v1_0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF" +} ---- To generate new API keys (requires admin access): [source,bash] ---- -# Using REST API (JAAS auth required) +# Using REST API (JAAS auth required). Replaces any existing key of the same type. curl -X POST "http://localhost:8181/cxs/tenants/mytenant/apikeys?type=PUBLIC&validityDays=30" \ - -u karaf:karaf \ - -H "Content-Type: application/json" + -u karaf:karaf -# Response (HTTP 200 OK): +# Response (HTTP 200 OK) — store plainTextKey immediately: { - "key": "8f7d9a2c-5e4b-3f1a-9b8c-7d6e5f4a3b2c", - "type": "PUBLIC", - "expirationDate": "2024-04-13T10:30:00Z", - "creationDate": "2024-03-14T10:30:00Z" + "apiKey": { + "maskedKey": "unomi_v1_****ab12", + "keyType": "PUBLIC", + "creationDate": "2024-03-14T10:30:00Z", + "expirationDate": "2024-04-13T10:30:00Z", + "revoked": false + }, + "plainTextKey": "unomi_v1_0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF" } # Using Karaf shell (requires admin access) @@ -392,37 +404,12 @@ unomi:crud create apikey -d '{"tenantId":"mytenant","keyType":"PUBLIC","validity There are three ways to authenticate with the Unomi API: -1. Tenant Authentication (Recommended for most endpoints): -[source,bash] ----- -# List all profiles (tenant access) -curl -X GET "http://localhost:8181/cxs/profiles" \ - --user "TENANT_ID:PRIVATE_KEY" \ - -H "Accept: application/json" - -# Response (HTTP 200 OK): -{ - "list": [ - { - "itemId": "profile1", - "properties": { - "firstName": "John", - "lastName": "Doe" - } - } - ], - "offset": 0, - "pageSize": 50, - "totalSize": 1 -} ----- -+ -2. Public API Access (Client-Side Operations): +1. Public API access (client-side / public endpoints): [source,bash] ---- # Get context data curl -X POST "http://localhost:8181/cxs/context.json" \ - -H "X-Unomi-Api-Key: 8f7d9a2c-5e4b-3f1a-9b8c-7d6e5f4a3b2c" \ + -H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ -H "Content-Type: application/json" \ -d '{ "source": { @@ -432,51 +419,39 @@ curl -X POST "http://localhost:8181/cxs/context.json" \ }, "requiredProfileProperties": ["firstName", "lastName"] }' - -# Response (HTTP 200 OK): -{ - "profileId": "xyz123", - "sessionId": "abc456", - "profileProperties": { - "firstName": "John", - "lastName": "Doe" - } -} ---- + -3. Private API Access (Server-Side Operations): +2. Tenant private-key authentication (most private REST endpoints): [source,bash] ---- -# Get profiles using tenant credentials -curl -X GET "http://localhost:8181/cxs/profiles" \ - --user "mytenant:1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6" \ - -H "Accept: application/json" - -# Response (HTTP 200 OK): -{ - "list": [ - { - "itemId": "profile1", - "scope": "mytenant", - "properties": { - "firstName": "John", - "lastName": "Doe" - } - } - ], +# Search profiles (there is no GET /cxs/profiles collection endpoint) +curl -X POST "http://localhost:8181/cxs/profiles/search" \ + --user "TENANT_ID:PRIVATE_KEY" \ + -H "Content-Type: application/json" \ + -H "Accept: application/json" \ + -d '{ + "condition": { + "type": "matchAllCondition" + }, "offset": 0, - "pageSize": 50, - "totalSize": 1 -} + "limit": 50 + }' +---- ++ +3. System administrator (JAAS) for tenant management and cluster operations: +[source,bash] +---- +curl -X GET "http://localhost:8181/cxs/tenants" \ + --user "karaf:karaf" \ + -H "Accept: application/json" ---- -Authentication Rules: - -1. If JAAS authentication is provided (username/password), it grants full access to all endpoints -2. Public paths (like /context.json) require a valid public API key -3. Private paths require both tenantId and private API key -4. All other requests are denied +Authentication rules (Unomi 3.1): +1. Public paths (for example `/cxs/context.json`, `/cxs/eventcollector`, `/cxs/client/*`) require a valid **public** API key via `X-Unomi-Api-Key` (unless <<_v2_compatibility_mode,V2 compatibility mode>> is enabled). +2. Private paths accept `tenantId:privateApiKey` Basic auth, or JAAS admin credentials (often with `X-Unomi-Tenant-Id` when a tenant context is required). +3. Tenant administration (`/cxs/tenants`) requires system administrator JAAS credentials. +4. JAAS admin does **not** replace the public API key on public endpoints. ==== Public vs Private Endpoints Public endpoints (requiring only public API key): @@ -846,7 +821,7 @@ curl -X POST 'http://localhost:8181/cxs/groovyActions' \ --form 'file=@helloWorldGroovyAction.groovy' ---- -Important: A bug ( https://issues.apache.org/jira/browse/UNOMI-847[UNOMI-847] ) in Apache Unomi 2.5 and lower requires the filename of a Groovy file being submitted to be the same than the id of the Groovy action (as per the example above). +Important: A bug ( https://issues.apache.org/jira/browse/UNOMI-847[UNOMI-847] ) in Apache Unomi 2.5 and lower requires the filename of a Groovy file being submitted to be the same as the id of the Groovy action (as per the example above). Finally, register a rule to trigger execution of the groovy action: [source,bash] @@ -1365,54 +1340,61 @@ ssh -p 8102 karaf@localhost or the user/password you have setup to protect the system if you have changed it. You can find the list of Apache Unomi shell commands in the "Shell commands" section of the documentation. -=== ElasticSearch authentication and security +=== Search engine authentication and security -With ElasticSearch 7, it's possible to secure the access to your data. (see https://www.elastic.co/guide/en/elasticsearch/reference/7.17/configuring-stack-security.html[https://www.elastic.co/guide/en/elasticsearch/reference/7.17/configuring-stack-security.html] and https://www.elastic.co/guide/en/elasticsearch/reference/7.17/secure-cluster.html[https://www.elastic.co/guide/en/elasticsearch/reference/7.17/secure-cluster.html]) +Apache Unomi 3.x talks to **Elasticsearch 9** or **OpenSearch 3** using the settings below. Prefer `etc/custom.system.properties` (or `unomi.custom.system.properties` / environment variables). The persistence OSGi configs (`etc/org.apache.unomi.persistence.elasticsearch.cfg` / `...opensearch.cfg`) expose the same values as short keys (`username`, `password`, `sslEnable`, `sslTrustAllCertificates`) that resolve from those system properties. -==== User authentication ! +Elasticsearch security reference: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-minimal-setup.html[Elasticsearch security]. +OpenSearch security reference: https://docs.opensearch.org/docs/latest/security/[OpenSearch Security]. -If your ElasticSearch have been configured to be only accessible by authenticated users, edit `etc/org.apache.unomi.persistence.elasticsearch.cfg` to add the following settings: +==== User authentication -[source] ----- -username=USER -password=PASSWORD ----- +If the search cluster requires authenticated users, set credentials in `custom.system.properties`: -==== SSL communication +[source,properties] +---- +# Elasticsearch +org.apache.unomi.elasticsearch.username=elastic +org.apache.unomi.elasticsearch.password=${env:ELASTIC_PASSWORD} -By default Unomi will communicate with ElasticSearch using `http` -but you can configure your ElasticSearch server(s) to allow encrypted request using `https`. +# OpenSearch (defaults often use admin + OPENSEARCH_INITIAL_ADMIN_PASSWORD) +org.apache.unomi.opensearch.username=admin +org.apache.unomi.opensearch.password=${env:OPENSEARCH_INITIAL_ADMIN_PASSWORD:-admin} +---- -You can follow this documentation to enable SSL on your ElasticSearch server(s): https://www.elastic.co/guide/en/elasticsearch/reference/7.17/security-basic-setup-https.html[https://www.elastic.co/guide/en/elasticsearch/reference/7.17/security-basic-setup-https.html] +Equivalent keys in the persistence `.cfg` files: `username=` / `password=`. -If your ElasticSearch is correctly configure to encrypt communications on `https`: +==== SSL communication -Just edit `etc/org.apache.unomi.persistence.elasticsearch.cfg` to add the following settings: +By default Unomi may use plain HTTP depending on the backend defaults (`org.apache.unomi.elasticsearch.sslEnable` defaults to `false`; `org.apache.unomi.opensearch.sslEnable` defaults to `true` in the package samples). To use HTTPS: -[source] +[source,properties] ---- -sslEnable=true +# Elasticsearch +org.apache.unomi.elasticsearch.sslEnable=true +org.apache.unomi.elasticsearch.sslTrustAllCertificates=true # development only + +# OpenSearch +org.apache.unomi.opensearch.sslEnable=true +org.apache.unomi.opensearch.sslTrustAllCertificates=true # development only; package default is true ---- -By default, certificates will have to be configured on the Apache Unomi server to be able to trust the identity -of the ElasticSearch server(s). But if you need to trust all certificates automatically, you can use this setting: +Environment variable equivalents include `UNOMI_ELASTICSEARCH_SSL_ENABLE`, `UNOMI_ELASTICSEARCH_SSL_TRUST_ALL_CERTIFICATES`, `UNOMI_OPENSEARCH_SSL_ENABLE`, and `UNOMI_OPENSEARCH_SSL_TRUST_ALL_CERTIFICATES`. -[source] ----- -sslTrustAllCertificates=true ----- +Equivalent keys in the persistence `.cfg` files: `sslEnable=` / `sslTrustAllCertificates=`. + +WARNING: `sslTrustAllCertificates=true` disables certificate validation. Use proper trust stores in production. ==== Permissions -Apache Unomi requires a particular set of Elasticsearch permissions for its operation. +Apache Unomi requires a particular set of search-engine permissions for its operation. -If you are using Elasticsearch in a production environment, you will most likely need to fine tune permissions given to the user used by Unomi. +If you are using Elasticsearch or OpenSearch in a production environment, you will most likely need to fine tune permissions given to the user used by Unomi. -The following permissions are required by Unomi: +The following privileges are required by Unomi: - required cluster privileges: `manage` OR `all` - - required index privileges on unomi indices: `write, manage, read` OR `all` + - required index privileges on Unomi indices: `write, manage, read` OR `all` === Unomi Distribution (features configuration) @@ -1455,7 +1437,7 @@ Be aware that in distribution's feature, you should only reference other feature mvn:org.apache.cxf.karaf/apache-cxf/${cxf.version}/xml/features mvn:org.apache.unomi/unomi-kar/${project.version}/xml/features - + unomi-base unomi-startup unomi-elasticsearch-core @@ -1493,7 +1475,7 @@ NOTE: For OpenSearch distributions, use `unomi-healthcheck-opensearch` instead o mvn:org.apache.cxf.karaf/apache-cxf/${cxf.version}/xml/features mvn:org.apache.unomi/unomi-kar/${project.version}/xml/features - + unomi-base unomi-startup unomi-elasticsearch-core @@ -1529,7 +1511,7 @@ NOTE: For OpenSearch distributions, use `unomi-healthcheck-opensearch` instead o mvn:org.apache.cxf.karaf/apache-cxf/${cxf.version}/xml/features mvn:org.apache.unomi/unomi-kar/${project.version}/xml/features - + unomi-base unomi-startup unomi-elasticsearch-core @@ -1754,7 +1736,7 @@ unomi:start elasticsearch-prod [#_health_check] === Health Check Extension -The Health Check extension provides a way to check is required Unomi components are 'live'. +The Health Check extension provides a way to check if required Unomi components are 'live'. It consists in a simple http endpoint that provide a JSON view of integrated health checks. It can then be used to determine if the server is up and running and can serve requests. @@ -1766,12 +1748,15 @@ to access the endpoint. Users and roles can be configured in the etc/users.prope Specific configuration is located in : org.apache.unomi.healthcheck.cfg Existing health checks are using configuration from that file, including authentication realm. -Existing health checks gives information about : -- Karaf (as soon as the karaf container is started, that check is LIVE) -- Elasticsearch (connection to elasticsearch cluster and its health) -- Unomi (unomi bundles status) -- Persistence (unomi to elasticsearch binding) -- Cluster health (unomi cluster status and nodes information) +Existing health checks give information about: + +* Karaf (as soon as the karaf container is started, that check is LIVE) +* Elasticsearch **or** OpenSearch (connection to the search cluster and its health) — provider name `elasticsearch` or `opensearch` depending on the distribution +* Unomi (unomi bundles status) +* Persistence (Unomi binding to the search/persistence backend) +* Cluster health (unomi cluster status and nodes information) + +Set `UNOMI_HEALTHCHECK_PROVIDERS` (or `providers` in the cfg file) to the comma-separated list that matches your deployment, for example `karaf,elasticsearch,unomi,persistence,cluster` or `karaf,opensearch,unomi,persistence,cluster`. All healthcheck can have a status : - DOWN (service is not available) @@ -1840,159 +1825,3 @@ By default, all healthcheck providers are included but the list of those include Karaf provider is the one needed by healthcheck (always LIVE), it cannot be ignored. The timeout used for each health check can be set by setting the property `timeout` to the desired value in milliseconds. An environment variable can be used to set this property : UNOMI_HEALTHCHECK_TIMEOUT - -=== API Access Examples - -1. Basic Authentication Example: -[source,bash] ----- -# Get authentication token -curl -X POST "http://localhost:8181/cxs/login" \ - -H "Content-Type: application/json" \ - -d '{ - "username": "myuser", - "password": "mypassword" - }' - -# Response (HTTP 200 OK): -{ - "token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...", - "totalSize": 1 -} ----- -+ -2. Public API Access (Client-Side Operations): -[source,bash] ----- -# Get context data -curl -X POST "http://localhost:8181/cxs/context.json" \ - -H "X-Unomi-Api-Key: 8f7d9a2c-5e4b-3f1a-9b8c-7d6e5f4a3b2c" \ - -H "Content-Type: application/json" \ - -d '{ - "source": { - "itemId": "homepage", - "itemType": "page", - "scope": "example" - }, - "requiredProfileProperties": ["firstName", "lastName"] - }' - -# Response (HTTP 200 OK): -{ - "profileId": "xyz123", - "sessionId": "abc456", - "profileProperties": { - "firstName": "John", - "lastName": "Doe" - } -} ----- -+ -3. Private API Access (Server-Side Operations): -[source,bash] ----- -# Get profiles using tenant credentials -curl -X GET "http://localhost:8181/cxs/profiles" \ - --user "mytenant:1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6" \ - -H "Accept: application/json" - -# Response (HTTP 200 OK): -{ - "list": [ - { - "itemId": "profile1", - "scope": "mytenant", - "properties": { - "firstName": "John", - "lastName": "Doe" - } - } - ], - "offset": 0, - "pageSize": 50, - "totalSize": 1 -} ----- - -Authentication Rules: - -1. If JAAS authentication is provided (username/password), it grants full access to all endpoints -2. Public paths (like /context.json) require a valid public API key -3. Private paths require both tenantId and private API key -4. All other requests are denied - -==== Public vs Private Endpoints - -Public endpoints (requiring only public API key): - -1. GET/POST /context.json -[source,bash] ----- -# Example request -curl -X GET "http://localhost:8181/cxs/context.json?sessionId=abc123" \ - -H "X-Unomi-Api-Key: 8f7d9a2c-5e4b-3f1a-9b8c-7d6e5f4a3b2c" ----- -+ -2. GET/POST /eventcollector -[source,bash] ----- -# Example request -curl -X POST "http://localhost:8181/cxs/eventcollector" \ - -H "X-Unomi-Api-Key: 8f7d9a2c-5e4b-3f1a-9b8c-7d6e5f4a3b2c" \ - -H "Content-Type: application/json" \ - -d '{ - "events": [{ - "eventType": "view", - "scope": "example", - "source": { - "itemId": "page1", - "itemType": "page", - "scope": "example" - }, - "target": { - "itemId": "product1", - "itemType": "product", - "scope": "example" - } - }] - }' ----- -+ -3. GET /client/* -[source,bash] ----- -# Example request -curl -X GET "http://localhost:8181/cxs/client/myapp/status" \ - -H "X-Unomi-Api-Key: 8f7d9a2c-5e4b-3f1a-9b8c-7d6e5f4a3b2c" ----- - -All other endpoints are considered private and require either: -* JAAS authentication with admin credentials, or -* Private API key authentication with tenant credentials - -Example private endpoint access: -[source,bash] ----- -# Get segment details -curl -X GET "http://localhost:8181/cxs/segments/important-customers" \ - --user "mytenant:1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6" \ - -H "Accept: application/json" - -# Create a new segment -curl -X POST "http://localhost:8181/cxs/segments" \ - --user "mytenant:1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6" \ - -H "Content-Type: application/json" \ - -d '{ - "itemId": "high-value-customers", - "name": "High Value Customers", - "description": "Customers with high purchase value", - "condition": { - "type": "profilePropertyCondition", - "parameterValues": { - "propertyName": "totalPurchases", - "comparisonOperator": "greaterThan", - "propertyValue": 1000 - } - } - }' ----- diff --git a/manual/src/main/asciidoc/connectors/salesforce-connector.adoc b/manual/src/main/asciidoc/connectors/salesforce-connector.adoc index 2ded160cce..9260397472 100644 --- a/manual/src/main/asciidoc/connectors/salesforce-connector.adoc +++ b/manual/src/main/asciidoc/connectors/salesforce-connector.adoc @@ -17,6 +17,8 @@ This connectors makes it possible to push and pull data to/from the Salesforce CRM. It can copy information between Apache Unomi profiles and Salesforce Leads. +NOTE: On Unomi **3.1**, profile and rule APIs used by the connector operate in a **tenant** context. Ensure the Unomi side uses tenant private-key or JAAS credentials with an explicit tenant (see <<_multitenancy,Multi-tenancy>>). The connector's Salesforce OAuth configuration is unchanged. + ==== Getting started ===== Salesforce account setup diff --git a/manual/src/main/asciidoc/context-request-flow.adoc b/manual/src/main/asciidoc/context-request-flow.adoc index 0747928ed7..8e023fbf02 100644 --- a/manual/src/main/asciidoc/context-request-flow.adoc +++ b/manual/src/main/asciidoc/context-request-flow.adoc @@ -15,4 +15,79 @@ Here is an overview of how Unomi processes incoming requests to the `ContextServlet`. -image::unomi-request.png[Unomi request overview] +[plantuml] +---- +@startuml +skinparam shadowing false +hide footbox + +participant ContextServlet +participant ProfileService +participant EventService +participant RulesService +participant PersistenceService + +[-> ContextServlet : Request + +== User identification == +ContextServlet -> ProfileService : Find/create user and session +ProfileService -> PersistenceService : Find/create user and session +PersistenceService --> ProfileService +ProfileService --> ContextServlet + +== Handle events == +loop all events + ContextServlet -> EventService : Send events + group Event handling + EventService -> RulesService : Call listener + RulesService -> RulesService : Get matching rules + loop all rules + RulesService -> PersistenceService : Test rule against current\nevent / source / profile / session + PersistenceService --> RulesService + loop all actions + RulesService -> RulesService : Execute action + end + RulesService -> EventService : Send 'rule fired' Event + note right of EventService + Recurse Event Handling + end note + end + RulesService -> EventService : Send profile updated event + note right of EventService + Recurse Event Handling + end note + end +end + +== Test condition filters == +loop all condition filters + ContextServlet -> ProfileService : Check condition against\ncurrent profile/session + ProfileService -> PersistenceService : Test condition against\ncurrent profile/session + PersistenceService --> ProfileService + ProfileService --> ContextServlet + note right of ContextServlet + Add filter results to answer + end note +end + +== Tracked Conditions == +ContextServlet -> RulesService : Get tracked conditions +loop all rules + RulesService -> PersistenceService : Test condition against\ncurrent event source + PersistenceService --> RulesService +end +RulesService --> ContextServlet +note right of ContextServlet + Add tracked conditions to answer +end note + +== Finalize == +ContextServlet -> ProfileService : Save profile and/or\nsession if needed +ProfileService -> PersistenceService : Save profile to persistence +PersistenceService --> ProfileService +ProfileService --> ContextServlet + +ContextServlet ->] : Response + +@enduml +---- diff --git a/manual/src/main/asciidoc/data-structures.adoc b/manual/src/main/asciidoc/data-structures.adoc index 0d4cf9a810..ca6a493463 100644 --- a/manual/src/main/asciidoc/data-structures.adoc +++ b/manual/src/main/asciidoc/data-structures.adoc @@ -23,9 +23,6 @@ All major entities in Apache Unomi inherit from two base classes that provide es @startuml skinparam componentStyle uml2 skinparam class { - BackgroundColor White - BorderColor DarkGray - ArrowColor DarkGray FontSize 14 } @@ -78,9 +75,6 @@ This diagram shows the key data structures in Apache Unomi and their relationshi @startuml skinparam componentStyle uml2 skinparam class { - BackgroundColor White - BorderColor DarkGray - ArrowColor DarkGray FontSize 14 } @@ -267,10 +261,6 @@ Conditions are the fundamental building blocks used across many Unomi components ---- @startuml skinparam componentStyle uml2 -skinparam component { - BackgroundColor LightBlue - BorderColor DarkBlue -} class Condition { + type: String @@ -319,10 +309,6 @@ Segments group profiles based on conditions. They are dynamic - profiles can ent ---- @startuml skinparam componentStyle uml2 -skinparam component { - BackgroundColor LightGreen - BorderColor DarkGreen -} class Segment { + itemId: String @@ -365,10 +351,6 @@ Rules define automated actions triggered by conditions. ---- @startuml skinparam componentStyle uml2 -skinparam component { - BackgroundColor LightYellow - BorderColor DarkGoldenRod -} class Rule { + itemId: String @@ -411,10 +393,6 @@ Goals track visitor progress toward specific objectives. ---- @startuml skinparam componentStyle uml2 -skinparam component { - BackgroundColor LightPink - BorderColor DarkRed -} class Goal { + itemId: String @@ -460,10 +438,6 @@ Campaigns organize marketing activities with goals, timeframes and costs. ---- @startuml skinparam componentStyle uml2 -skinparam component { - BackgroundColor LightSalmon - BorderColor DarkRed -} class Campaign { + itemId: String @@ -507,10 +481,6 @@ This diagram shows how these components work together in practice. ---- @startuml skinparam componentStyle uml2 -skinparam component { - BackgroundColor White - BorderColor Black -} actor Visitor participant "Event Collector" as EC @@ -556,9 +526,6 @@ The following diagram illustrates how conditions can be nested and composed to c @startuml skinparam componentStyle uml2 skinparam class { - BackgroundColor White - BorderColor DarkGray - ArrowColor DarkGray FontSize 14 } @@ -635,11 +602,6 @@ Here are some examples of how conditions can be composed: ---- @startuml skinparam componentStyle uml2 -skinparam object { - BackgroundColor White - BorderColor DarkGray - ArrowColor DarkGray -} object "BooleanCondition (AND)" as root { type = "booleanCondition" @@ -706,11 +668,6 @@ Past event conditions have a special structure that allows for complex temporal ---- @startuml skinparam componentStyle uml2 -skinparam object { - BackgroundColor White - BorderColor DarkGray - ArrowColor DarkGray -} object "PastEventCondition" as past { type = "pastEventCondition" diff --git a/manual/src/main/asciidoc/datamodel.adoc b/manual/src/main/asciidoc/datamodel.adoc index f22571bd2f..2dc1164ec1 100755 --- a/manual/src/main/asciidoc/datamodel.adoc +++ b/manual/src/main/asciidoc/datamodel.adoc @@ -22,7 +22,207 @@ user profiles into segments along user-definable dimensions or acted upon by alg The following data model only contains the classes and properties directly related to the most important objects of Apache Unomi. There are other classes that are less central to the functionality but all the major ones are represented in the diagram below: -image::data-model.png[] +[plantuml] +---- +@startuml +skinparam shadowing false +skinparam classAttributeIconSize 0 +hide circle +hide empty methods + +class Item { + scope: String + itemType: String + version: Long + itemId: String +} + +class Metadata { + id: String + name: String + description: String + scope: String + enabled: boolean + hidden: boolean + readOnly: boolean + missingPlugins: boolean + tags: Set + systemTags: Set +} + +class MetadataItem { + metadata: Metadata + scope: String +} + +class CustomItem { + customItemType: String + properties: Map +} + +class TimestampedItem { + timeStamp: Date +} + +class Profile { + scope: String + mergedWith: String + anonymousProfile: boolean + properties: Map + systemProperties: Map + scores: Map + segments: Set + consents: Map +} + +class Persona +class Consent { + scope: String + typeIdentifier: String + status: ConsentStatus + statusDate: Date + revokeDate: Date + consentGrantedNow: boolean +} + +enum ConsentStatus + +class Session { + scope: String + profileId: String + profile: Profile + timeStamp: Date + lastEventDate: Date + duration: int + size: int + properties: Map + systemProperties: Map + originEventIds: List + originEventTypes: List +} + +class Event { + eventType: String + sessionId: String + profileId: String + session: Session + profile: Profile + source: Item + target: Item + timeStamp: Date + persistent: boolean + properties: Map + attributes: Map + flattenedProperties: Map +} + +class ConditionType { + conditionEvaluator: String + queryBuilder: String + parameters: List + parentCondition: Condition +} + +class Condition { + conditionTypeId: String + conditionType: ConditionType + parameterValues: Map +} + +class Parameter { + id: String + type: String + defaultValue: String + multivalued: boolean + choiceListInitializerFilter: String +} + +class ActionType { + actionExecutor: String + parameters: List +} + +class Action { + actionTypeId: String + actionType: ActionType + parameterValues: Map +} + +class Rule { + condition: Condition + actions: List + linkedItems: List + priority: int + raiseEventOnlyOnce: boolean + raiseEventOnlyOnceForProfile: boolean + raiseEventOnlyOnceForSession: boolean +} + +class Segment { + condition: Condition +} + +class Scoring { + elements: List +} + +class Campaign { + startDate: Date + endDate: Date + timezone: String + currency: String + cost: Double + primaryGoal: String + entryCondition: Condition +} + +class Goal { + campaignId: String + startEvent: Condition + targetEvent: Condition +} + +class Scope +class UserList + +Item <|-- MetadataItem +Item <|-- CustomItem +Item <|-- Profile +Item <|-- Session +TimestampedItem <|-- Event +MetadataItem <|-- ConditionType +MetadataItem <|-- ActionType +MetadataItem <|-- Rule +MetadataItem <|-- Segment +MetadataItem <|-- Scoring +MetadataItem <|-- Campaign +MetadataItem <|-- Goal +MetadataItem <|-- Scope +MetadataItem <|-- UserList +Profile <|-- Persona + +MetadataItem "1" *-- "1" Metadata +Profile "1" *-- "*" Consent +Consent "1" --> "1" ConsentStatus +Session "*" --> "1" Profile +Event "*" --> "1" Session +Event "*" --> "1" Profile +Event --> Item : source +Event --> Item : target +Rule "1" *-- "1" Condition +Rule "1" *-- "*" Action +Condition --> ConditionType +Action --> ActionType +ConditionType "*" *-- "*" Parameter +ActionType "*" *-- "*" Parameter +Segment --> Condition +Campaign --> Condition : entryCondition +Goal --> Condition : startEvent +Goal --> Condition : targetEvent +Campaign "1" o-- "*" Goal + +@enduml +---- We will detail many of these classes in the document below. @@ -813,8 +1013,8 @@ You can think of a rule as a structure that looks like this: Basically when a rule is evaluated, all the conditions in the `when` part are evaluated and if the result matches (meaning it evaluates to `true`) then the actions will be executed in sequence. -The real power of Apache Unomi comes from the fact that `conditions` and `actions` are fully pluggeable and that plugins may implement new conditions and/or actions to perform any task. -You can imagine conditions checking incoming event data against third-party systems or even against authentication systesm, and actions actually pulling or pushing data to third-party systems. +The real power of Apache Unomi comes from the fact that `conditions` and `actions` are fully pluggable and that plugins may implement new conditions and/or actions to perform any task. +You can imagine conditions checking incoming event data against third-party systems or even against authentication systems, and actions actually pulling or pushing data to third-party systems. For example the Salesforce CRM connector is simply a set of actions that pull and push data into the CRM. It is then just a matter of setting up the proper rules with the proper conditions to determine when and how the data will be pulled or pushed into the third-party system. diff --git a/manual/src/main/asciidoc/event-processing.adoc b/manual/src/main/asciidoc/event-processing.adoc index 84780aeaa0..f1a8931916 100644 --- a/manual/src/main/asciidoc/event-processing.adoc +++ b/manual/src/main/asciidoc/event-processing.adoc @@ -17,11 +17,6 @@ ---- @startuml skinparam componentStyle uml2 -skinparam component { - BackgroundColor<> LightBlue - BackgroundColor<> LightGreen - BackgroundColor<> LightYellow -} package "Event Processing" { [Event Service] <> diff --git a/manual/src/main/asciidoc/getting-started.adoc b/manual/src/main/asciidoc/getting-started.adoc index ac9fe5b093..223cb38486 100644 --- a/manual/src/main/asciidoc/getting-started.adoc +++ b/manual/src/main/asciidoc/getting-started.adoc @@ -37,8 +37,8 @@ them at your own risks. Apache Unomi supports two search engine backends: -* *ElasticSearch*: Version **9.x** (for example 9.4.3) is required for Unomi 3.0 and later. Version 7.17.5 applies only to Unomi 2.x. -* *OpenSearch*: Version 3.x is supported starting with Unomi 3.1 +* *Elasticsearch*: Version **9.x** (for example 9.4.3) is required for Unomi 3.0 and later. Version 7.17.5 applies only to Unomi 2.x. +* *OpenSearch*: Version **3.7.0** (matching `opensearch.version` in the Unomi root POM) is supported starting with Unomi 3.1 It is highly recommended to use the versions specified in the documentation. When in doubt, consult the Apache Unomi community for the latest compatibility information. @@ -85,7 +85,15 @@ curl -X POST http://localhost:8181/cxs/tenants \ }' ---- -Save the API keys from the response - you'll need them for all subsequent API calls. See <<_multitenancy,Multi-tenancy>> for authentication details. +The tenant create response includes **masked** API key metadata only. Regenerate keys to obtain one-time `plainTextKey` values: + +[source,bash] +---- +curl -X POST "http://localhost:8181/cxs/tenants/default/apikeys?type=PUBLIC" --user karaf:karaf +curl -X POST "http://localhost:8181/cxs/tenants/default/apikeys?type=PRIVATE" --user karaf:karaf +---- + +Store `plainTextKey` immediately — you'll need the public key for subsequent API calls. See <<_multitenancy,Multi-tenancy>> for authentication details. Now you can: diff --git a/manual/src/main/asciidoc/graphql-examples.adoc b/manual/src/main/asciidoc/graphql-examples.adoc index 4d0537ac35..847a7ac99c 100644 --- a/manual/src/main/asciidoc/graphql-examples.adoc +++ b/manual/src/main/asciidoc/graphql-examples.adoc @@ -175,7 +175,17 @@ To make this query work you need to supply authorization token in the `HTTP head } ---- -NOTE: When using curl, you can use the `--user` option instead of manually encoding credentials. For example, `--user karaf:karaf` automatically handles Base64 encoding for Basic authentication. +NOTE: GraphQL requests need authentication. For **mutations and administrative queries**, use HTTP Basic with JAAS (`karaf:karaf`) or `tenantId:privateApiKey`. For **public read** operations on the `cdp` root field, send `X-Unomi-Api-Key` with a tenant public API key. See <<_graphql_api,GraphQL API>> authentication. + +When using curl, you can use the `--user` option instead of manually encoding credentials. For example, `--user karaf:karaf` or `--user TENANT_ID:PRIVATE_KEY` automatically handles Base64 encoding for Basic authentication. For public reads: + +[source,bash] +---- +curl -X POST http://localhost:8181/graphql \ + -H "Content-Type: application/json" \ + -H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ + -d '{"query":"{ cdp { ... } }"}' +---- The result will now show the list of profiles: diff --git a/manual/src/main/asciidoc/images/data-model.png b/manual/src/main/asciidoc/images/data-model.png deleted file mode 100644 index e63d771a4f..0000000000 Binary files a/manual/src/main/asciidoc/images/data-model.png and /dev/null differ diff --git a/manual/src/main/asciidoc/images/expression-filtering-layers.png b/manual/src/main/asciidoc/images/expression-filtering-layers.png index 6164f48c4d..432faaf5c0 100644 Binary files a/manual/src/main/asciidoc/images/expression-filtering-layers.png and b/manual/src/main/asciidoc/images/expression-filtering-layers.png differ diff --git a/manual/src/main/asciidoc/images/process-creation-extension.png b/manual/src/main/asciidoc/images/process-creation-extension.png deleted file mode 100644 index 0c7357a853..0000000000 Binary files a/manual/src/main/asciidoc/images/process-creation-extension.png and /dev/null differ diff --git a/manual/src/main/asciidoc/images/process-creation-schema.png b/manual/src/main/asciidoc/images/process-creation-schema.png deleted file mode 100644 index 4158eb50b2..0000000000 Binary files a/manual/src/main/asciidoc/images/process-creation-schema.png and /dev/null differ diff --git a/manual/src/main/asciidoc/images/profile-alias-example.png b/manual/src/main/asciidoc/images/profile-alias-example.png index aa418513a5..38b3233af7 100644 Binary files a/manual/src/main/asciidoc/images/profile-alias-example.png and b/manual/src/main/asciidoc/images/profile-alias-example.png differ diff --git a/manual/src/main/asciidoc/images/profile-alias-external-ids.png b/manual/src/main/asciidoc/images/profile-alias-external-ids.png index 6e20e56b53..d09fcd6749 100644 Binary files a/manual/src/main/asciidoc/images/profile-alias-external-ids.png and b/manual/src/main/asciidoc/images/profile-alias-external-ids.png differ diff --git a/manual/src/main/asciidoc/images/profile-alias-overview.png b/manual/src/main/asciidoc/images/profile-alias-overview.png index db41defdf8..4b863fb062 100644 Binary files a/manual/src/main/asciidoc/images/profile-alias-overview.png and b/manual/src/main/asciidoc/images/profile-alias-overview.png differ diff --git a/manual/src/main/asciidoc/images/unomi-request.png b/manual/src/main/asciidoc/images/unomi-request.png deleted file mode 100755 index fdba277024..0000000000 Binary files a/manual/src/main/asciidoc/images/unomi-request.png and /dev/null differ diff --git a/manual/src/main/asciidoc/images/unomi-rule-engine.png b/manual/src/main/asciidoc/images/unomi-rule-engine.png index c65dbd890f..65a4bcaab6 100644 Binary files a/manual/src/main/asciidoc/images/unomi-rule-engine.png and b/manual/src/main/asciidoc/images/unomi-rule-engine.png differ diff --git a/manual/src/main/asciidoc/index.adoc b/manual/src/main/asciidoc/index.adoc index 5cda165886..c14d4b5f46 100644 --- a/manual/src/main/asciidoc/index.adoc +++ b/manual/src/main/asciidoc/index.adoc @@ -23,6 +23,7 @@ Apache Software Foundation :homepage: https://unomi.apache.org :docinfo: shared-head,shared-footer :source-highlighter: highlightjs +:plantuml-config: {docdir}/plantuml/unomi-theme.puml ifndef::backend-pdf[] [%passthrough,subs=attributes] @@ -162,7 +163,25 @@ include::scheduler.adoc[] [#_health_check_chapter] == Health check -The health check endpoint and providers are documented in the Configuration chapter: <<_health_check,Health check extension>>. +Apache Unomi exposes an HTTP health endpoint at `/health/check` with pluggable providers. + +Typical providers: + +* `karaf` — always required (Karaf framework) +* `elasticsearch` or `opensearch` — search backend (match your distribution) +* `persistence` — persistence binding +* `cluster` — cluster node registry +* `unomi` — Unomi bundles / services + +Docker / env example (OpenSearch distribution): + +[source,bash] +---- +UNOMI_HEALTHCHECK_ENABLED=true +UNOMI_HEALTHCHECK_PROVIDERS=cluster,opensearch,unomi,persistence +---- + +The endpoint is protected by the `health` role (default user `health` / `health`). Full provider configuration, sample JSON, and extension points are documented in the Configuration chapter: <<_health_check,Health check extension>>. == Reference diff --git a/manual/src/main/asciidoc/javascript-tracker-guide.adoc b/manual/src/main/asciidoc/javascript-tracker-guide.adoc index 044945bd87..c1179e909a 100644 --- a/manual/src/main/asciidoc/javascript-tracker-guide.adoc +++ b/manual/src/main/asciidoc/javascript-tracker-guide.adoc @@ -11,6 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // +[#_javascript_tracker_guide] === Implementing a JavaScript Tracker This guide explains how to implement a basic JavaScript tracker for Apache Unomi from scratch. This will help you understand the underlying concepts and API interactions, whether you're building a custom tracker or integrating Apache Unomi into an existing application. diff --git a/manual/src/main/asciidoc/jsonSchema/extend-an-existing-schema.adoc b/manual/src/main/asciidoc/jsonSchema/extend-an-existing-schema.adoc index 6a147577dd..8cebec8cc4 100644 --- a/manual/src/main/asciidoc/jsonSchema/extend-an-existing-schema.adoc +++ b/manual/src/main/asciidoc/jsonSchema/extend-an-existing-schema.adoc @@ -80,7 +80,28 @@ With this extension the property *myNewProp* can now be added to the event. Process when adding extension: -image::process-creation-extension.png[pdfwidth=35%,align=center] +[plantuml] +---- +@startuml +skinparam shadowing false +hide footbox + +actor User +actor "Authenticated user" as AuthUser +participant UNOMI +participant Elasticsearch + +UNOMI -> UNOMI : Start timer to load\nextensions in memory + +AuthUser -> UNOMI : Send extension +UNOMI -> Elasticsearch : Persist extension +UNOMI -> UNOMI : Load extension in memory + +User -> UNOMI : Send an event +UNOMI -> UNOMI : Get the schema with extension\nand validate the event against it + +@enduml +---- ==== How to add an extension through the API diff --git a/manual/src/main/asciidoc/jsonSchema/introduction.adoc b/manual/src/main/asciidoc/jsonSchema/introduction.adoc index 046d145912..b8a987e37a 100644 --- a/manual/src/main/asciidoc/jsonSchema/introduction.adoc +++ b/manual/src/main/asciidoc/jsonSchema/introduction.adoc @@ -14,7 +14,7 @@ === Introduction -Introduced with Apache Unomi 2.0, JSON-Schema are used to validate data submitted through all of the public (unprotected) API endpoints. +Introduced with Apache Unomi 2.0, JSON Schema is used to validate data submitted through public API endpoints (for example `/cxs/context.json`). In Unomi 3.1 those endpoints require a tenant public API key (`X-Unomi-Api-Key`); they are no longer anonymous. ==== What is a JSON Schema @@ -278,5 +278,25 @@ Schemas persisted in Elasticsearch do not require a restart of the platform to r Process of creation of schemas: -image::process-creation-schema.png[pdfwidth=35%,align=center] +[plantuml] +---- +@startuml +skinparam shadowing false +hide footbox + +actor Admin +actor "Authenticated user" as AuthUser +participant UNOMI +participant Elasticsearch + +Admin -> UNOMI : Start unomi +UNOMI -> UNOMI : Load predefined schemas in memory +UNOMI -> UNOMI : Start timer to load\npersisted schemas in memory + +AuthUser -> UNOMI : Send schema +UNOMI -> Elasticsearch : Persist schema +UNOMI -> UNOMI : Load schema in memory + +@enduml +---- diff --git a/manual/src/main/asciidoc/migrations/migrate-1.4-to-1.5.adoc b/manual/src/main/asciidoc/migrations/migrate-1.4-to-1.5.adoc index c4dbf70644..51a8a9c6ae 100644 --- a/manual/src/main/asciidoc/migrations/migrate-1.4-to-1.5.adoc +++ b/manual/src/main/asciidoc/migrations/migrate-1.4-to-1.5.adoc @@ -22,7 +22,7 @@ To be able to do so, we had to rework the way the data was stored inside Elastic Previously every items was stored inside the same ElasticSearch index but this is not allowed anymore in recent ElasticSearch versions. -Since Apache Unomi version 1.5.0 every type of items (see section: link:#_items[Items]) is now stored in a dedicated separated index. +Since Apache Unomi version 1.5.0 every type of items (see section: <<_item,Item>>) is now stored in a dedicated separated index. ===== API changes diff --git a/manual/src/main/asciidoc/migrations/migrate-1.6-to-2.0.adoc b/manual/src/main/asciidoc/migrations/migrate-1.6-to-2.0.adoc index fff03cefe5..06e824650d 100644 --- a/manual/src/main/asciidoc/migrations/migrate-1.6-to-2.0.adoc +++ b/manual/src/main/asciidoc/migrations/migrate-1.6-to-2.0.adoc @@ -154,7 +154,7 @@ The following environment variables are used for the migration: |=== -If there is a need for advanced configuratiion, the configuration file used by Apache Unomi 2.0 is located in: `etc/org.apache.unomi.migration.cfg` +If there is a need for advanced configuration, the configuration file used by Apache Unomi 2.0 is located in: `etc/org.apache.unomi.migration.cfg` ===== Migrate manually diff --git a/manual/src/main/asciidoc/migrations/migrate-2.x-to-3.0.adoc b/manual/src/main/asciidoc/migrations/migrate-2.x-to-3.0.adoc index 5dd4e2da1e..37e2df7ba2 100644 --- a/manual/src/main/asciidoc/migrations/migrate-2.x-to-3.0.adoc +++ b/manual/src/main/asciidoc/migrations/migrate-2.x-to-3.0.adoc @@ -235,14 +235,14 @@ While the mapping system provides backward compatibility, it is recommended to u ==== Configuration Changes -===== OpenSearch Security Configuration +===== OpenSearch Security Configuration (Unomi 3.1+) -When using OpenSearch 3.x with Apache Unomi 3.1, security is enabled by default and requires specific configuration: +OpenSearch as a Unomi backend is supported starting with **Apache Unomi 3.1** (not required for a pure 2.x → 3.0 upgrade on Elasticsearch). When using OpenSearch 3.x with Unomi 3.1, security is commonly enabled and requires: [source,properties] ---- -# OpenSearch security settings (required by default since OpenSearch 3) -org.apache.unomi.opensearch.ssl.enable=true +# OpenSearch security settings (sslEnable defaults to true in package samples) +org.apache.unomi.opensearch.sslEnable=true org.apache.unomi.opensearch.username=admin org.apache.unomi.opensearch.password=${env:OPENSEARCH_INITIAL_ADMIN_PASSWORD:-admin} org.apache.unomi.opensearch.sslTrustAllCertificates=true diff --git a/manual/src/main/asciidoc/migrations/migrate-3.0-to-3.1.adoc b/manual/src/main/asciidoc/migrations/migrate-3.0-to-3.1.adoc index a107ce7a72..18b69e2880 100644 --- a/manual/src/main/asciidoc/migrations/migrate-3.0-to-3.1.adoc +++ b/manual/src/main/asciidoc/migrations/migrate-3.0-to-3.1.adoc @@ -229,22 +229,36 @@ Before starting the migration, please ensure that: - You do have a backup of your data - You did practice the migration in a staging environment, NEVER migrate a production environment without prior validation -- You verified your applications were operational with Apache Unomi 3.1 (authentication updated, client applications updated, ...) - You are currently running Apache Unomi 3.0 (or a later 3.0.x version) - You understand the multi-tenancy impact on your data model -- You have configured tenant-specific API keys for your applications +- You have a plan to update client applications to tenant API keys (or temporary <<_v2_compatibility_mode,V2 compatibility mode>> only if coming from 2.x) +- You know how to obtain plaintext API keys after upgrade (regenerate via `/cxs/tenants/{id}/apikeys`; create responses expose masked keys only) === Migration Process -The migration from 3.0 to 3.1 is primarily a configuration and authentication update: +Upgrading from 3.0 to 3.1 requires a **data migration** (bundled Groovy scripts) plus client authentication updates. Contributors maintaining those scripts should follow <<_writing_migration_scripts,Writing migration scripts>> (step tracking, idempotency, and common pitfalls). -Contributors maintaining the Groovy scripts that implement this upgrade should follow <<_writing_migration_scripts,Writing migration scripts>> (step tracking, idempotency, and common pitfalls). - -1. **Shutdown your Apache Unomi 3.0 cluster** -2. **Update your client applications** to use the new authentication model -3. **Configure tenant-specific API keys** for your applications -4. **Start your Apache Unomi 3.1 cluster** -5. **Test your applications** with the new authentication model +1. **Shut down your Apache Unomi 3.0 cluster** (migration runs with Unomi stopped; the search engine must remain reachable). +2. **Install Apache Unomi 3.1** on the node(s) that will run the migration (Karaf shell available). +3. **Run the migration** from the Karaf shell: ++ +[source,bash] +---- +unomi:migrate 3.0.0 +---- ++ +Pass your actual origin version without qualifier (for example `3.0.0`). Optional second argument skips the confirmation prompt. You can preconfigure `etc/org.apache.unomi.migration.cfg` instead of answering interactive questions; see <<_shell_commands,Shell commands>>. ++ +Scripts applied for the 3.1 line include (under `tools/shell-commands/.../META-INF/cxs/migration/`): ++ +* `migrate-3.1.0-00-fixProfileNbOfVisits` +* `migrate-3.1.0-01-tenantDocumentIds` +* `migrate-3.1.0-05-fixSystemItemIds` +* `migrate-3.1.0-10-tenantInitialization` (creates the default tenant used for isolation) +* `migrate-3.1.0-15-updateLegacyQueryBuilder` +4. **Update client applications** to the 3.1 authentication model (public key header / tenant private key), or enable <<_v2_compatibility_mode,V2 compatibility mode>> only when migrating **from Unomi 2.x**. +5. **Start your Apache Unomi 3.1 cluster**. +6. **Regenerate and store tenant API keys** (plaintext is returned only from the key-creation endpoint), then **test** applications end to end. === Transitional authentication options diff --git a/manual/src/main/asciidoc/migrations/migrate-elasticsearch-to-opensearch.adoc b/manual/src/main/asciidoc/migrations/migrate-elasticsearch-to-opensearch.adoc index b68e29ac39..b01c019d84 100644 --- a/manual/src/main/asciidoc/migrations/migrate-elasticsearch-to-opensearch.adoc +++ b/manual/src/main/asciidoc/migrations/migrate-elasticsearch-to-opensearch.adoc @@ -14,200 +14,121 @@ [#_migrate_from_elasticsearch_to_opensearch] ==== Migrating from Elasticsearch to OpenSearch -:toc: macro -:toclevels: 4 -:toc-title: Table of contents -:numbered: -toc::[] +Apache Unomi 3.1+ can use OpenSearch instead of Elasticsearch as its search/persistence backend. +Moving existing data is a **search-engine cutover**: copy Unomi indices from Elasticsearch to OpenSearch, then point Unomi at the OpenSearch distribution. +It is **not** performed by `unomi:migrate` (that command upgrades Unomi data shape between Unomi versions). -===== Overview - -This guide describes how to migrate your Apache Unomi data from Elasticsearch to OpenSearch. The migration process involves using the OpenSearch Replication Tool, which is designed to handle large-scale migrations efficiently while maintaining data consistency. +This page only covers Unomi-specific index naming and cutover steps. +Use the official OpenSearch documentation for how to copy cluster data; tooling and supported version matrices change over time. ===== Prerequisites -Before starting the migration, ensure you have: +* Apache Unomi already running against Elasticsearch (typically Elasticsearch 9.x for Unomi 3.x) +* A target OpenSearch cluster at the version Unomi expects (see `opensearch.version` in the Unomi root POM; currently **3.7.0**) +* Network path between the clusters (or shared snapshot storage, depending on the method you choose) +* Backup of the Elasticsearch indices, and a staging rehearsal before production +* Plan a write freeze (stop or quiesce Unomi) for the final cutover so source and target do not diverge -* Running Elasticsearch cluster with your Unomi data -* Target OpenSearch cluster set up and running -* Sufficient disk space on the target cluster -* Java 17 or later installed -* Network connectivity between source and target clusters +===== Unomi index naming -===== Migration Options +By default Unomi stores data under the index prefix `context` +(`org.apache.unomi.elasticsearch.index.prefix` / `org.apache.unomi.opensearch.index.prefix`). -====== Option 1: OpenSearch Replication Tool (Recommended) +Examples: -The OpenSearch Replication Tool is the recommended approach for production environments, especially for large datasets. +* `context-profile`, `context-segment`, `context-rule`, … +* Rollover types: `context-event-*`, `context-session-*`, … -====== Installation +List what you must copy: [source,bash] ---- -git clone https://github.com/opensearch-project/opensearch-migrations.git -cd opensearch-migrations/replication-tool -./gradlew build +curl -s "http://SOURCE:9200/_cat/indices/context-*?v" ---- -====== Configuration +If you customized the prefix, replace `context` with your value everywhere below, and keep the **same** prefix on the OpenSearch side after cutover. -Create a configuration file `config.yml`: +Also migrate **aliases** and any ILM/ISM (or equivalent) policies that Unomi relies on for rollover indices. Official migration tooling does not always move policies automatically; check the tool’s component support list. -[source,yaml] ----- -source: - hosts: ["source-elasticsearch-host:9200"] - user: "elastic_user" # if authentication is enabled - password: "elastic_pass" # if authentication is enabled - -destination: - hosts: ["target-opensearch-host:9200"] - user: "opensearch_user" # if authentication is enabled - password: "opensearch_pass" # if authentication is enabled - -indices: - - name: "context-*" # Unomi context indices - - name: "segment-*" # Unomi segment indices - - name: "profile-*" # Unomi profile indices - - name: "session-*" # Unomi session indices ----- +===== Choose a data-copy method (external docs) -====== Running the Migration +Pick a method that matches your Elasticsearch/OpenSearch versions, downtime budget, and ops stack. +Always confirm the current **supported source → target matrix** before you start. -[source,bash] ----- -./bin/replication-tool --config config.yml ----- +[cols="1,2",options="header"] +|=== +| Approach | Where to start -The tool provides progress updates and ensures data consistency during the migration. +| Migration Assistant (metadata, snapshot backfill / Reindex-from-Snapshot, optional live Capture and Replay) +| https://docs.opensearch.org/latest/migration-assistant/[Migration Assistant for OpenSearch] · +https://docs.opensearch.org/latest/migration-assistant/is-migration-assistant-right-for-you/[Is it right for you? (version matrix)] · +https://github.com/opensearch-project/opensearch-migrations[opensearch-migrations on GitHub] -====== Option 2: Logstash Pipeline +| Remote reindex into OpenSearch (copy documents from a remote Elasticsearch cluster) +| https://docs.opensearch.org/latest/im-plugin/reindex-data/[Reindex data] · +https://docs.opensearch.org/latest/api-reference/document-apis/reindex/[Reindex API] + +(create destination indices with mappings/settings first; remote host must be allowlisted) -For smaller deployments or when more control over the migration process is needed, you can use Logstash. +| Logstash (scroll from Elasticsearch, ship to OpenSearch) +| https://docs.opensearch.org/latest/tools/logstash/[Logstash and OpenSearch] · +https://docs.opensearch.org/latest/tools/logstash/ship-to-opensearch/[Ship events to OpenSearch] +(`logstash-output-opensearch` plugin) +|=== -====== Logstash Configuration +TIP: For a self-managed Unomi deployment with planned downtime, **remote reindex** is often the simplest path once destination mappings exist. +For large or near-zero-downtime migrations, evaluate **Migration Assistant** against the published matrix for your Elasticsearch major version (Unomi 3.x commonly uses Elasticsearch **9.x**; confirm support in the upstream docs). -Create a file named `logstash-migration.conf`: +===== Verify the copy -[source,ruby] ----- -input { - elasticsearch { - hosts => ["source-elasticsearch-host:9200"] - index => "context-*" # Repeat for other indices - size => 5000 - scroll => "5m" - docinfo => true - user => "elastic_user" # if authentication is enabled - password => "elastic_pass" # if authentication is enabled - } -} - -output { - opensearch { - hosts => ["target-opensearch-host:9200"] - index => "%{[@metadata][_index]}" - document_id => "%{[@metadata][_id]}" - user => "opensearch_user" # if authentication is enabled - password => "opensearch_pass" # if authentication is enabled - } -} ----- +Before switching Unomi: -====== Running Logstash Migration +* Compare `_cat/indices` document counts for every Unomi index (and rollover generations) +* Confirm aliases used by rollover types still resolve +* Spot-check a few documents by id [source,bash] ---- -logstash -f logstash-migration.conf +curl -s "http://SOURCE:9200/_cat/indices/context-*?v&h=index,docs.count" +curl -s "https://TARGET:9200/_cat/indices/context-*?v&h=index,docs.count" ---- -===== Post-Migration Steps +(Adjust scheme, auth, and TLS as required by each cluster.) -1. Verify Data Integrity +===== Point Unomi at OpenSearch + +1. Stop Apache Unomi (or otherwise freeze writes). +2. Switch the runtime distribution to OpenSearch: + [source,bash] ---- -# Check document counts -curl -X GET "source-elasticsearch-host:9200/_cat/indices/context-*?v" -curl -X GET "target-opensearch-host:9200/_cat/indices/context-*?v" ----- +# Docker / environment +UNOMI_DISTRIBUTION=unomi-distribution-opensearch -2. Update Unomi Configuration -+ -Edit `etc/custom.system.properties`: +# Or from the Karaf shell before starting Unomi features +unomi:setup -d=unomi-distribution-opensearch -f=true +---- +3. Point Unomi at the target cluster in `etc/custom.system.properties` (or `unomi.custom.system.properties` / env vars). Example: + [source,properties] ---- -# Comment out or remove Elasticsearch properties -#org.apache.unomi.elasticsearch.addresses=localhost:9200 -#org.apache.unomi.elasticsearch.cluster.name=contextElasticSearch +# org.apache.unomi.elasticsearch.* (disable / stop using) -# Add OpenSearch properties org.apache.unomi.opensearch.addresses=localhost:9200 -org.apache.unomi.opensearch.cluster.name=contextOpenSearch -org.apache.unomi.opensearch.sslEnable=false +org.apache.unomi.opensearch.cluster.name=opensearch-cluster +org.apache.unomi.opensearch.index.prefix=context org.apache.unomi.opensearch.username=admin -org.apache.unomi.opensearch.password=admin +org.apache.unomi.opensearch.password=${env:OPENSEARCH_INITIAL_ADMIN_PASSWORD:-admin} +org.apache.unomi.opensearch.sslEnable=true +org.apache.unomi.opensearch.sslTrustAllCertificates=true ---- - -3. Restart Apache Unomi + -[source,bash] ----- -./bin/stop -./bin/start ----- - -===== Troubleshooting - -====== Common Issues - -1. Connection Timeouts -* Increase the timeout settings in your configuration -* Check network connectivity between clusters - -2. Memory Issues -* Adjust JVM heap size for the migration tool -* Consider reducing batch sizes - -3. Missing Indices -* Verify index patterns in configuration -* Check source cluster health - -====== Monitoring Progress - -The OpenSearch Replication Tool provides progress information during migration: - -* Documents copied -* Time elapsed -* Current transfer rate -* Estimated completion time - -===== Best Practices - -1. *Testing* -* Always test the migration process in a non-production environment first -* Verify all Unomi features work with migrated data - -2. *Performance* -* Run migration during off-peak hours -* Monitor system resources during migration -* Use appropriate batch sizes based on document size - -3. *Backup* -* Create backups of your Elasticsearch indices before migration -* Keep source cluster running until verification is complete - -4. *Validation* -* Compare document counts between source and target -* Verify index mappings and settings -* Test Unomi functionality with migrated data +Align `sslEnable` / trust settings with your OpenSearch security setup. See <<_configuration,Configuration>>. +4. Start Unomi and verify health, tenant/API access, and a representative profile/event flow. -===== Support +Keep the Elasticsearch cluster until you are satisfied the OpenSearch deployment is correct, then decommission it. -For additional support: +===== Further help -* OpenSearch Replication Tool: https://github.com/opensearch-project/opensearch-migrations -* Apache Unomi Community: https://unomi.apache.org/community.html -* OpenSearch Forum: https://forum.opensearch.org/ \ No newline at end of file +* Apache Unomi community: https://unomi.apache.org/community.html +* OpenSearch forum: https://forum.opensearch.org/ diff --git a/manual/src/main/asciidoc/migrations/migrate-es7-to-es9.adoc b/manual/src/main/asciidoc/migrations/migrate-es7-to-es9.adoc index 8e03b788fb..53d175636a 100644 --- a/manual/src/main/asciidoc/migrations/migrate-es7-to-es9.adoc +++ b/manual/src/main/asciidoc/migrations/migrate-es7-to-es9.adoc @@ -20,7 +20,7 @@ To execute the migration, you should have one Elasticsearch 7 running (your sour This upgrade relies on a script. If you are sharing the Elasticsearch instance with other projects, it might need to be adjusted. -The script migration_es7-es9.sh at the root of the project and handles: +The script `migration_es7-es9.sh` at the root of the project handles: * Regular indices and rollover indices with their aliases * ILM policies migration * Data reindexing from ES7 to ES9 @@ -33,7 +33,7 @@ The script migration_es7-es9.sh at the root of the project and handles: * `curl` for HTTP requests * Access to both ES7 (source) and ES9 (destination) clusters * *ES9 must have `reindex.remote.whitelist` configured* (see configuration below) -* Ensure the machine where ES9 is running have access to the ES7 environment +* Ensure the machine where ES9 is running has access to the ES7 environment Install `jq` if not already installed: @@ -73,6 +73,9 @@ export ES9_USER="elastic" export ES9_PASSWORD="your-es9-password" export INDEX_PREFIX="context-" +# Script default includes a trailing hyphen and concatenates patterns (context-profile, …). +# Unomi system property org.apache.unomi.*.index.prefix is the prefix *without* a hyphen (`context`); +# the persistence layer adds `-` when forming index names. export BATCH_SIZE="1000" ---- @@ -89,7 +92,7 @@ export BATCH_SIZE="1000" | ES9_HOST | Elasticsearch 9 URL | http://localhost:9201 | ES9_USER | ES9 username | elastic | ES9_PASSWORD | ES9 password | password -| INDEX_PREFIX | Prefix for index names | context- +| INDEX_PREFIX | Prefix for index names (script concatenates; include trailing `-`) | context- | BATCH_SIZE | Reindex batch size | 1000 |=== diff --git a/manual/src/main/asciidoc/migrations/migrations.adoc b/manual/src/main/asciidoc/migrations/migrations.adoc index b5a24d4c64..9cbe001462 100644 --- a/manual/src/main/asciidoc/migrations/migrations.adoc +++ b/manual/src/main/asciidoc/migrations/migrations.adoc @@ -14,6 +14,39 @@ This section contains information and steps to migrate between major Unomi versions. +Use this decision guide to pick the right runbook: + +* **Unomi 2.x → 3.0** (platform): <<_migrate_from_2_x_to_3_0,Migrate from 2.x to 3.0>> (+ <<_migrate_from_elasticsearch_7_to_elasticsearch_9,ES7→ES9>> if needed) +* **Unomi 3.0 → 3.1** (tenants / API keys): <<_migrate_from_3_0_to_3_1,Migrate from 3.0 to 3.1>> (`unomi:migrate`); optional <<_v2_compatibility_mode,V2 compatibility mode>> for 2.x clients +* **Elasticsearch → OpenSearch** (same Unomi version, backend swap): <<_migrate_from_elasticsearch_to_opensearch,Migrate from Elasticsearch to OpenSearch>> (not `unomi:migrate`) + +[plantuml] +---- +@startuml +skinparam shadowing false +title Migration decision (Unomi 3.x) + +start +if (Changing Unomi version?) then (yes) + if (From 2.x?) then (yes) + :2.x to 3.0 platform upgrade; + :ES7 to ES9 data copy if needed; + if (Need multi-tenancy / API keys?) then (yes) + :3.0 to 3.1 via unomi:migrate; + :Optional V2 compatibility mode; + endif + else (from 3.0) + :3.0 to 3.1 via unomi:migrate; + endif +else (no) +endif +if (Switching Elasticsearch to OpenSearch?) then (yes) + :Backend cutover guide\n(not unomi:migrate); +endif +stop +@enduml +---- + include::writing-migration-scripts.adoc[] === V2/V3 API Compatibility Guide diff --git a/manual/src/main/asciidoc/migrations/v2-compatibility-mode.adoc b/manual/src/main/asciidoc/migrations/v2-compatibility-mode.adoc index 4a7e54f306..0614d523bf 100644 --- a/manual/src/main/asciidoc/migrations/v2-compatibility-mode.adoc +++ b/manual/src/main/asciidoc/migrations/v2-compatibility-mode.adoc @@ -17,11 +17,11 @@ This document explains how to use the V2 compatibility mode in Apache Unomi V3, which allows V2 client applications to work with Unomi V3 without requiring API keys. -== Overview +==== Overview The V2 compatibility mode is designed to ease the migration from Unomi V2 to V3 by allowing V2 clients to continue working without immediate changes to their authentication logic. This mode provides backward compatibility while still leveraging the multi-tenant architecture of V3. -=== How It Works +===== How It Works When V2 compatibility mode is enabled: @@ -33,7 +33,7 @@ When V2 compatibility mode is enabled: This allows V2 clients to work with Unomi V3 immediately after migration, giving you time to gradually update client applications to use the new V3 authentication model. -== Prerequisites +==== Prerequisites Before enabling V2 compatibility mode, ensure that: @@ -41,9 +41,9 @@ Before enabling V2 compatibility mode, ensure that: 2. **Default Tenant Exists**: A default tenant exists that will be used for all operations 3. **V3 Installation**: Unomi V3 is properly installed and configured -== Configuration +==== Configuration -=== Enable V2 Compatibility Mode +===== Enable V2 Compatibility Mode You can also set the environment variable `UNOMI_REST_AUTHENTICATION_V2COMPATIBILITYMODEENABLED=true` (maps to `org.apache.unomi.rest.authentication.v2CompatibilityModeEnabled` in `custom.system.properties`). @@ -71,25 +71,24 @@ You can also set the environment variable `UNOMI_REST_AUTHENTICATION_V2COMPATIBI ./bin/start ``` -=== Configuration Management +===== Configuration Management V2 compatibility mode is managed through configuration files only. This approach is safer and prevents accidental changes to authentication settings. -== Migration Workflow +==== Migration Workflow -=== Step 1: Migrate Data +===== Step 1: Migrate Data -First, migrate your V2 data to V3 using the migration scripts: +First, migrate your data with Apache Unomi stopped using the single shell command (bundled Groovy scripts under `META-INF/cxs/migration/` run automatically for versions after the origin you pass): ```bash -# Run the migration scripts -unomi:migrate-3.1.0-00-tenantDocumentIds -unomi:migrate-3.1.0-10-tenantInitialization +# From the Karaf shell (Unomi runtime stopped). Example when upgrading from 2.x through 3.1: +unomi:migrate 2.0.0 ``` -The `migrate-3.1.0-10-tenantInitialization` script creates a default tenant that will be used for V2 compatibility mode. +The `migrate-3.1.0-10-tenantInitialization` script (run as part of that chain when migrating through 3.1) creates a default tenant used for V2 compatibility mode. See <<_migrate_from_3_0_to_3_1,Migrate from 3.0 to 3.1>> and <<_shell_commands,Shell commands>>. -=== Step 2: Enable V2 Compatibility Mode +===== Step 2: Enable V2 Compatibility Mode Enable V2 compatibility mode by updating the configuration file: @@ -98,14 +97,14 @@ Enable V2 compatibility mode by updating the configuration file: vi etc/org.apache.unomi.rest.authentication.cfg # Set v2.compatibilitymode.enabled = true -# Set v2CompatibilityDefaultTenantId = your-tenant-id +# Set v2.compatibilitymode.defaultTenantId = your-tenant-id # Restart the server to apply changes ./bin/stop ./bin/start ``` -=== Step 3: Test V2 Clients +===== Step 3: Test V2 Clients Your V2 clients should now work without any changes: @@ -122,7 +121,7 @@ RestAssured.given() .post("/context.json"); ``` -=== Step 4: Gradual Migration +===== Step 4: Gradual Migration Over time, gradually update your clients to use V3 authentication: @@ -130,9 +129,9 @@ Over time, gradually update your clients to use V3 authentication: 2. **Test with V3 authentication** while keeping V2 compatibility mode enabled 3. **Disable V2 compatibility mode** once all clients are updated -== Client Migration Examples +==== Client Migration Examples -=== From V2 to V3 (with V2 Compatibility Mode) +===== From V2 to V3 (with V2 Compatibility Mode) **V2 Client (continues to work)**: ```java @@ -157,16 +156,16 @@ given() .post("/context.json"); ``` -=== Gradual Migration Strategy +===== Gradual Migration Strategy 1. **Phase 1**: Enable V2 compatibility mode, V2 clients continue working 2. **Phase 2**: Develop and test V3 clients alongside V2 clients 3. **Phase 3**: Migrate clients one by one to V3 authentication 4. **Phase 4**: Disable V2 compatibility mode once all clients are migrated -== Security Considerations +==== Security Considerations -=== V2 Compatibility Mode Security +===== V2 Compatibility Mode Security When V2 compatibility mode is enabled: @@ -176,7 +175,7 @@ When V2 compatibility mode is enabled: - **All operations** use the default tenant context - **Non-protected events** require no authentication (same as V2) -=== Protected Events in V2 Compatibility Mode +===== Protected Events in V2 Compatibility Mode In V2 compatibility mode, protected event types are configured dynamically using the V2 third-party configuration file. By default, the following event types are protected: @@ -191,7 +190,7 @@ For protected events, clients must: All other event types are considered non-protected and require no authentication. -=== V2 Third-Party Configuration +===== V2 Third-Party Configuration The protected events and third-party providers are configured in the original V2 configuration file `etc/org.apache.unomi.thirdparty.cfg`. The system dynamically detects any number of providers using the pattern `thirdparty.{providerName}.{property}`: @@ -209,7 +208,7 @@ thirdparty.myapp.allowedEvents=${org.apache.unomi.thirdparty.myapp.allowedEvents This uses the exact same configuration format as V2, ensuring complete compatibility with existing V2 setups. The system automatically detects and configures any provider that has a valid key. -=== Configuration Management +===== Configuration Management The V2 third-party configuration supports dynamic updates: @@ -243,21 +242,21 @@ The V2 third-party configuration supports dynamic updates: ./bin/start ``` -=== Recommendations +===== Recommendations 1. **Use V2 compatibility mode temporarily** during migration 2. **Plan for gradual migration** to V3 authentication 3. **Monitor access patterns** during the transition 4. **Disable V2 compatibility mode** once migration is complete -== Troubleshooting +==== Troubleshooting -=== Common Issues +===== Common Issues **V2 clients still not working**: - Check configuration file: `etc/org.apache.unomi.rest.authentication.cfg` - Verify `v2.compatibilitymode.enabled = true` -- Ensure `v2CompatibilityDefaultTenantId` matches the tenant ID used during migration +- Ensure `v2.compatibilitymode.defaultTenantId` matches the tenant ID used during migration - Ensure the tenant exists and is accessible **Authentication errors**: @@ -270,7 +269,7 @@ The V2 third-party configuration supports dynamic updates: - Verify tenant exists in the tenant index - Check tenant configuration in the migration scripts -=== Debugging +===== Debugging Enable debug logging for authentication: @@ -286,7 +285,7 @@ Check authentication filter logs: log:display | grep AuthenticationFilter ``` -== Disabling V2 Compatibility Mode +==== Disabling V2 Compatibility Mode Once all clients are migrated to V3 authentication: @@ -305,11 +304,11 @@ Once all clients are migrated to V3 authentication: 4. **Monitor for any issues** and address them before final deployment -== Testing V2 Compatibility Mode +==== Testing V2 Compatibility Mode The existing test framework supports testing V2 compatibility mode using system properties. -=== Running Tests in V2 Compatibility Mode +===== Running Tests in V2 Compatibility Mode To run tests with V2 compatibility mode enabled: @@ -322,7 +321,7 @@ export UNOMI_V2_COMPATIBILITY_MODE=true mvn test ``` -=== Test Framework Integration +===== Test Framework Integration The test framework automatically detects V2 compatibility mode and uses the appropriate client: @@ -331,7 +330,7 @@ The test framework automatically detects V2 compatibility mode and uses the appr This allows you to test both V2 compatibility mode and normal V3 mode using the same test suite. -=== Example Test Execution +===== Example Test Execution ```bash # Test with V2 compatibility mode (server should be configured for V2 compatibility) @@ -341,7 +340,7 @@ mvn test -Dunomi.v2.compatibility.mode=true -Dunomi.url=http://localhost:8181 mvn test -Dunomi.url=http://localhost:8181 ``` -== Conclusion +==== Conclusion The V2 compatibility mode provides a smooth migration path from Unomi V2 to V3, allowing you to: diff --git a/manual/src/main/asciidoc/migrations/v2-v3-compatibility.adoc b/manual/src/main/asciidoc/migrations/v2-v3-compatibility.adoc index 1c7a729884..4eb64ca280 100644 --- a/manual/src/main/asciidoc/migrations/v2-v3-compatibility.adoc +++ b/manual/src/main/asciidoc/migrations/v2-v3-compatibility.adoc @@ -12,17 +12,17 @@ // limitations under the License. // -== Apache Unomi V2/V3 API Differences Guide +=== Apache Unomi V2/V3 API Differences Guide This document explains the key differences between Apache Unomi 2.x and 3.x versions from an API perspective. -== Overview +==== Overview -Apache Unomi 3.x introduces comprehensive multi-tenancy support, enabling complete data isolation between different tenants. This fundamental architectural change requires a new tenant-based authentication model while keeping all API endpoints unchanged. +Apache Unomi **3.1** introduces comprehensive multi-tenancy support, enabling complete data isolation between different tenants. This fundamental architectural change requires a new tenant-based authentication model while keeping all API endpoints unchanged. -=== Multi-Tenancy in V3 +===== Multi-Tenancy in 3.1 -The key innovation in V3 is the introduction of **tenant isolation** for all data: +The key innovation in Unomi 3.1 is the introduction of **tenant isolation** for all data: - **Profiles, events, segments, rules, and schemas** are now tenant-specific - **Complete data separation** between tenants - no cross-tenant data access @@ -31,13 +31,13 @@ The key innovation in V3 is the introduction of **tenant isolation** for all dat This multi-tenancy support necessitates the authentication changes described below, as the system must now identify which tenant context to operate in for every request. -== Key Differences Between V2 and V3 +==== Key Differences Between V2 and V3 -=== Authentication Model +===== Authentication Model [cols="1,1,1", options="header"] |=== -|Aspect |Unomi V2 |Unomi V3 +|Aspect |Unomi V2 |Unomi 3.1 |Authentication Method |System Administrator Authentication (karaf/karaf) @@ -56,14 +56,14 @@ This multi-tenancy support necessitates the authentication changes described bel |System Administrator Authentication (karaf/karaf) |=== -=== API Key Types (V3 Only) +===== API Key Types (V3 Only) V3 introduces two types of API keys per tenant: - **Public Key**: Used for public endpoints (event collection via `/context.json`) - **Private Key**: Used with tenantId for tenant-specific administrative operations -=== Authentication Requirements by Endpoint Type +===== Authentication Requirements by Endpoint Type [cols="1,1,1", options="header"] |=== @@ -82,7 +82,7 @@ V3 introduces two types of API keys per tenant: |System Admin (karaf/karaf) |=== -== Authentication Flow (V3) +==== Authentication Flow (V3) The AuthenticationFilter in V3 follows this resolution order: @@ -92,9 +92,9 @@ The AuthenticationFilter in V3 follows this resolution order: - **Tenant Authentication**: Basic Auth with `tenantId:privateKey` - **System Administrator Authentication**: Basic Auth with `karaf:karaf` (or configured admin credentials) -== Code Examples +==== Code Examples -=== V2 Authentication +===== V2 Authentication [source,java] ---- @@ -110,7 +110,7 @@ RestAssured.given() .post("/context.json"); ---- -=== V3 Authentication +===== V3 Authentication [source,java] ---- @@ -143,9 +143,9 @@ given() .post("/cxs/tenants"); ---- -== Implementation Strategy +==== Implementation Strategy -=== Client Factory Pattern +===== Client Factory Pattern [source,java] ---- @@ -162,7 +162,7 @@ public class UnomiConfiguration { } ---- -=== Version-Specific Authentication +===== Version-Specific Authentication [source,java] ---- @@ -184,9 +184,9 @@ public void updateKeys(String publicKey, String privateKey) { } ---- -== Migration Guidelines +==== Migration Guidelines -=== From V2 to V3 +===== From V2 to V3 1. **Understand Multi-Tenancy Impact** - All data (profiles, events, segments, rules, schemas) becomes tenant-specific @@ -209,7 +209,7 @@ public void updateKeys(String publicKey, String privateKey) { - Request/response payloads are unchanged - Only authentication mechanism differs -=== Benefits of Multi-Tenancy in V3 +===== Benefits of Multi-Tenancy in V3 - **Data Isolation**: Complete separation ensures tenant data never crosses boundaries - **Scalability**: Support for multiple customers/organizations in a single Unomi instance @@ -217,7 +217,7 @@ public void updateKeys(String publicKey, String privateKey) { - **Compliance**: Easier to meet data privacy regulations with clear tenant boundaries - **Cost Efficiency**: Shared infrastructure with isolated data reduces operational costs -== Conclusion +==== Conclusion The fundamental difference between Unomi V2 and V3 is the introduction of **comprehensive multi-tenancy support**: diff --git a/manual/src/main/asciidoc/migrations/writing-migration-scripts.adoc b/manual/src/main/asciidoc/migrations/writing-migration-scripts.adoc index 64f543ae03..3ef6bee9a6 100644 --- a/manual/src/main/asciidoc/migrations/writing-migration-scripts.adoc +++ b/manual/src/main/asciidoc/migrations/writing-migration-scripts.adoc @@ -18,7 +18,7 @@ This section is for **contributors** who add or change Groovy migration scripts shipped in the `shell-commands` module. Operators upgrading a cluster should follow the version-specific guides in this chapter instead. -Migration scripts run with **Apache Unomi stopped**, via `unomi:migrate` (see <>). +Migration scripts run with **Apache Unomi stopped**, via `unomi:migrate` (see <<_shell_commands,Shell commands>>). They talk directly to Elasticsearch or OpenSearch over HTTP and transform persisted data in place. ==== Script location and naming diff --git a/manual/src/main/asciidoc/multitenancy.adoc b/manual/src/main/asciidoc/multitenancy.adoc index e91c30e294..6f6a914cee 100644 --- a/manual/src/main/asciidoc/multitenancy.adoc +++ b/manual/src/main/asciidoc/multitenancy.adoc @@ -17,11 +17,11 @@ Apache Unomi 3.1+ isolates profiles, events, segments, rules, and schemas per tenant. Each tenant has public and private API keys. See also <<_migrate_from_3_0_to_3_1,Migrate from 3.0 to 3.1>> for authentication changes. -== Overview +==== Overview Apache Unomi provides robust multi-tenancy support, allowing multiple organizations to use the same Unomi instance while maintaining complete data isolation. Each tenant gets their own dedicated space with separate data storage, configuration, and API keys. -== Key Features +==== Key Features * Complete data isolation between tenants * Dual API key system (public and private keys) @@ -30,9 +30,9 @@ Apache Unomi provides robust multi-tenancy support, allowing multiple organizati * Migration tools for existing data * Support for REST APIs (and GraphQL when enabled) -== Authentication Methods +==== Authentication Methods -=== Public API Key +===== Public API Key Used for public endpoints (e.g., context requests) that are typically accessed from client-side applications. [source,http] @@ -40,7 +40,7 @@ Used for public endpoints (e.g., context requests) that are typically accessed f X-Unomi-Api-Key: ---- -=== Private API Key +===== Private API Key Used for administrative operations and sensitive endpoints. Requires Basic Authentication using tenant ID and private key. [source,bash] @@ -50,9 +50,9 @@ Used for administrative operations and sensitive endpoints. Requires Basic Authe NOTE: curl automatically handles Base64 encoding when using the `--user` option, so you don't need to manually encode the credentials. -== Getting Started +==== Getting Started -=== Creating Your First Tenant +===== Creating Your First Tenant To create a new tenant, use the Tenant API endpoint: @@ -70,34 +70,61 @@ curl -X POST http://localhost:8181/cxs/tenants \ }' ---- -The response includes the tenant with automatically generated API keys: +The create response includes the tenant with **masked** API key metadata only (plaintext secrets are not returned and cannot be recovered from storage): [source,json] ---- { "itemId": "my-tenant", - "name": "My Organization", - "description": "My organization description", + "itemType": "tenant", + "status": "ACTIVE", "apiKeys": [ { - "type": "PUBLIC", - "key": "abc123...", - "created": "2024-01-01T00:00:00Z" + "maskedKey": "unomi_v1_****ab12", + "keyType": "PUBLIC", + "revoked": false }, { - "type": "PRIVATE", - "key": "xyz789...", - "created": "2024-01-01T00:00:00Z" + "maskedKey": "unomi_v1_****cd34", + "keyType": "PRIVATE", + "revoked": false } - ] + ], + "properties": { + "name": "My Organization", + "description": "My organization description" + } } ---- -NOTE: Both public and private API keys are automatically generated when creating a tenant. Extract the `key` value from the appropriate API key object in the `apiKeys` array. +IMPORTANT: Store plaintext keys immediately after calling the key creation endpoint. Creating a tenant generates keys server-side, but only masked values appear on the tenant resource. Regenerate (replace) keys to obtain `plainTextKey`: + +[source,bash] +---- +curl -X POST "http://localhost:8181/cxs/tenants/my-tenant/apikeys?type=PUBLIC" \ + --user karaf:karaf + +curl -X POST "http://localhost:8181/cxs/tenants/my-tenant/apikeys?type=PRIVATE" \ + --user karaf:karaf +---- + +Example response: + +[source,json] +---- +{ + "apiKey": { + "maskedKey": "unomi_v1_****ab12", + "keyType": "PUBLIC", + "revoked": false + }, + "plainTextKey": "unomi_v1_0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF" +} +---- -=== Making Your First API Call +===== Making Your First API Call -==== Public Endpoint Example (Context Request) +====== Public Endpoint Example (Context Request) [source,bash] ---- @@ -114,7 +141,7 @@ curl http://localhost:8181/cxs/context.json \ }' ---- -==== Private Endpoint Example (Profile Management) +====== Private Endpoint Example (Profile Management) [source,bash] ---- @@ -123,9 +150,9 @@ curl -X GET http://localhost:8181/cxs/profiles \ -H "Content-Type: application/json" ---- -== Configuration +==== Configuration -=== Basic Setup +===== Basic Setup Configure default tenant settings in `etc/org.apache.unomi.tenant.cfg`: @@ -145,7 +172,7 @@ tenant.apikey.rotation.warning.days=7 tenant.security.roles.prefix=unomi_tenant_ ---- -=== Security Provider Configuration +===== Security Provider Configuration For Elasticsearch: [source,properties] @@ -159,9 +186,9 @@ For OpenSearch: tenant.security.provider=opensearch ---- -== Security Model +==== Security Model -=== Roles and Permissions +===== Roles and Permissions Apache Unomi implements a hierarchical role-based access control (RBAC) system. The main roles are: @@ -197,11 +224,11 @@ The configuration uses the format `operation.roles.OPERATION_NAME=ROLE1,ROLE2,.. - Multiple roles are comma-separated - Changes take effect immediately without restart -=== Operation Configuration +===== Operation Configuration Operations in Unomi can be customized to require specific roles. This is configured through OSGi configuration files. -==== Configuration File +====== Configuration File Create or modify the file `etc/org.apache.unomi.security.cfg`: @@ -228,7 +255,7 @@ The configuration uses the format `operation.roles.OPERATION_NAME=ROLE1,ROLE2,.. - Multiple roles are comma-separated - Changes take effect immediately without restart -==== Common Operations +====== Common Operations Here are some common operations and their typical role requirements: @@ -244,7 +271,7 @@ Here are some common operations and their typical role requirements: |RULE_UPDATE |Update business rules |ROLE_UNOMI_TENANT_ADMIN |=== -==== Custom Operations +====== Custom Operations To define custom operations: @@ -270,7 +297,7 @@ public void performCustomOperation() { } ---- -=== Subjects and Authentication +===== Subjects and Authentication A Subject represents an authenticated entity in the system. There are three types of subjects: @@ -298,7 +325,7 @@ tenantSubject.getPrincipals().add(new UserPrincipal("tenant-id")); tenantSubject.getPrincipals().add(new RolePrincipal("ROLE_UNOMI_TENANT_ADMIN")); ---- -=== Tenant-Role Relationship +===== Tenant-Role Relationship Each tenant has associated public and private roles: @@ -310,7 +337,7 @@ Each tenant has associated public and private roles: * Full access to tenant data * Can perform all tenant operations -=== Operation Validation +===== Operation Validation The security service validates operations based on: @@ -331,7 +358,7 @@ securityService.executeAsSystemSubject(() -> { }); ---- -=== Best Practices +===== Best Practices 1. Role Assignment: * Assign minimum required roles @@ -353,9 +380,9 @@ securityService.executeAsSystemSubject(() -> { * Validate operations before execution * Maintain proper audit trails -== Tenant Management +==== Tenant Management -=== Listing Tenants +===== Listing Tenants [source,bash] ---- @@ -364,7 +391,7 @@ curl -X GET http://localhost:8181/cxs/tenants \ -H "Content-Type: application/json" ---- -=== Updating a Tenant +===== Updating a Tenant [source,bash] ---- @@ -377,22 +404,22 @@ curl -X PUT http://localhost:8181/cxs/tenants/my-tenant \ }' ---- -=== Regenerating API Keys +===== Regenerating API Keys + +Regenerating replaces any existing key of the same type. Store `plainTextKey` from the response immediately. [source,bash] ---- -curl -X POST http://localhost:8181/cxs/tenants/my-tenant/apikeys \ - --user karaf:karaf \ - -H "Content-Type: application/json" \ - -d '{ - "type": "PUBLIC", - "validityDays": 30 - }' +curl -X POST "http://localhost:8181/cxs/tenants/my-tenant/apikeys?type=PUBLIC&validityDays=30" \ + --user karaf:karaf + +curl -X POST "http://localhost:8181/cxs/tenants/my-tenant/apikeys?type=PRIVATE" \ + --user karaf:karaf ---- -NOTE: You can specify `type` as `PUBLIC` or `PRIVATE`, and optionally set `validityDays` for key expiration. If omitted, both keys will be regenerated. +NOTE: `type` must be `PUBLIC` or `PRIVATE`. Optional `validityDays` sets expiration; omit it (or use `0`) for no expiration. -=== Deleting a Tenant +===== Deleting a Tenant [source,bash] ---- @@ -401,7 +428,7 @@ curl -X DELETE http://localhost:8181/cxs/tenants/my-tenant \ -H "Content-Type: application/json" ---- -== GraphQL Support +==== GraphQL Support GraphQL endpoints support both public and private authentication methods: @@ -415,7 +442,7 @@ curl -X POST http://localhost:8181/graphql \ }' ---- -=== Tenant-Specific GraphQL Schemas +===== Tenant-Specific GraphQL Schemas The GraphQL API provides tenant-specific schemas, meaning each tenant can have a unique GraphQL schema based on their property types and configurations. This ensures that tenants only see the data and fields relevant to their specific implementation. @@ -428,9 +455,9 @@ When a tenant accesses the GraphQL API: For complete details on the GraphQL multi-tenancy implementation, refer to the <<_graphql_api,GraphQL API>> section of the documentation. -== Monitoring and Management +==== Monitoring and Management -=== Monitoring API Usage +===== Monitoring API Usage Unomi exposes read-only usage metrics per tenant. Quota enforcement belongs in your upstream gateway or control plane. @@ -445,7 +472,7 @@ Supported `period` values are `current-month` (default), `YYYY-MM` (for example The response includes profile, scope, segment, and rule totals, per-scope segment/rule breakdown (`scopeUsages`), monthly `eventCount` for the requested period (`periodStart` / `periodEnd` in epoch millis), active API key count, storage document count, in-memory REST request count since process start, and `collectedAt` (epoch millis). Values refresh on a background schedule and may be stale until the next collection cycle. -=== Event retention purge +===== Event retention purge Upstream control planes can delete old tenant events through Unomi instead of talking to the search cluster directly: @@ -458,56 +485,56 @@ curl -X POST "http://localhost:8181/cxs/tenants/my-tenant/purge/events?retention The response reports how many events matched the retention cutoff before deletion ran (`eventsMatched`) and whether the delete-by-query completed successfully (`purgeRequested`); a `purgeRequested` of `false` means the deletion failed and the request returns HTTP 500 (see server logs for the cause). The minimum accepted retention is seven days, to guard against accidentally purging recent or active event data. -=== Data Migration +===== Data Migration -Migrate data between tenants: +There is no REST endpoint to copy data between tenants. Version upgrades that introduce or adjust multi-tenancy run through the Karaf shell with Unomi stopped: [source,bash] ---- -curl -X POST http://localhost:8181/cxs/tenants/source-tenant/migrate/target-tenant \ - --user karaf:karaf \ - -H "Content-Type: application/json" +unomi:migrate 3.0.0 ---- -== Best Practices +See <<_migrate_from_3_0_to_3_1,Migrate from 3.0 to 3.1>> and <<_shell_commands,Shell commands>>. + +==== Best Practices -=== API Key Management +===== API Key Management * Rotate keys regularly using the key regeneration endpoint * Use public keys only for public endpoints * Never expose private keys in client-side code * Monitor API key usage and implement rate limiting -=== Resource Management +===== Resource Management * Set appropriate quotas for each tenant * Monitor resource usage through the tenant usage API (`GET /cxs/tenants/{tenantId}/usage`) * Configure alerts for quota limits * Regularly review and adjust limits based on usage patterns -=== Security +===== Security * Always use HTTPS in production * Implement proper key rotation policies * Conduct regular security audits * Monitor for suspicious activity patterns * Keep tenant configurations up to date -== Troubleshooting +==== Troubleshooting -=== Common Issues +===== Common Issues -==== 401 Unauthorized +====== 401 Unauthorized * Verify API key is correct * Check if using public key for private endpoint * Ensure tenant ID matches the API key -==== 400 Bad Request +====== 400 Bad Request * Check if API key header is present * Verify request format is correct -==== 404 Not Found +====== 404 Not Found * Verify tenant ID exists * Check if endpoint path is correct -=== Logging +===== Logging Enable debug logging for tenant-related operations: @@ -516,47 +543,43 @@ Enable debug logging for tenant-related operations: log4j.logger.org.apache.unomi.tenant=DEBUG ---- -== Migration Guide +==== Migration Guide -=== Migrating Existing Data +===== Migrating Existing Data -To migrate existing data to use multi-tenancy: +To attach existing 3.0 data to the multi-tenant model when upgrading to 3.1: +1. Stop Apache Unomi. +2. Run the bundled migration from the Karaf shell: ++ [source,bash] ---- -# Step 1: Create new tenant -curl -X POST http://localhost:8181/cxs/tenants \ - --user karaf:karaf \ - -H "Content-Type: application/json" \ - -d '{ - "requestedId": "new-tenant", - "properties": { - "name": "New Tenant", - "description": "Migrated tenant" - } - }' - -# Step 2: Migrate data -curl -X POST http://localhost:8181/cxs/tenants/migration/default/new-tenant \ - --user karaf:karaf \ - -H "Content-Type: application/json" +unomi:migrate 3.0.0 ---- ++ +This runs the `migrate-3.1.0-*` Groovy scripts (tenant document IDs, system item IDs, default tenant initialization, legacy queryBuilder IDs, and related fixes). See <<_migrate_from_3_0_to_3_1,Migrate from 3.0 to 3.1>>. +3. Start Unomi 3.1 and regenerate tenant API keys if you need the plaintext secrets (create/migrate responses only expose masked keys). See <<_multitenancy,Creating Your First Tenant>> / API key regeneration below. -=== Verification +===== Verification -After migration, verify data integrity: +After migration, verify tenant context and data access: [source,bash] ---- -# Check profile count -curl -X GET http://localhost:8181/cxs/tenants/new-tenant/profiles/count \ +# List tenants (JAAS admin) +curl -X GET http://localhost:8181/cxs/tenants \ --user karaf:karaf \ - -H "Content-Type: application/json" + -H "Accept: application/json" + +# List profiles for a tenant (tenant private key auth) +curl -X GET http://localhost:8181/cxs/profiles \ + --user "TENANT_ID:PRIVATE_KEY" \ + -H "Accept: application/json" ---- -== Working with Events and Rules +==== Working with Events and Rules -=== Creating Custom Event Types +===== Creating Custom Event Types First, create a JSON schema for your custom event type and deploy it using the JSON schema endpoint: @@ -656,7 +679,7 @@ curl -X POST http://localhost:8181/cxs/jsonSchema/validateEvent \ }' ---- -=== Sending Custom Events +===== Sending Custom Events Once the event type is defined, you can send events: @@ -692,7 +715,7 @@ curl -X POST http://localhost:8181/cxs/context.json \ }' ---- -=== Creating Rules for Event Processing +===== Creating Rules for Event Processing Create a rule to update profile properties based on purchase events: @@ -735,7 +758,7 @@ curl -X POST http://localhost:8181/cxs/rules \ }' ---- -=== Testing the Event Processing +===== Testing the Event Processing To test that everything works: @@ -793,9 +816,9 @@ Expected response will show updated properties: } ---- -=== Advanced Rule Examples +===== Advanced Rule Examples -==== Segmenting High-Value Customers +====== Segmenting High-Value Customers Create a segment for customers with high total revenue: @@ -821,7 +844,7 @@ curl -X POST http://localhost:8181/cxs/segments \ }' ---- -==== Tracking Purchase Frequency +====== Tracking Purchase Frequency Create a rule to track days between purchases: diff --git a/manual/src/main/asciidoc/past-event-conditions.adoc b/manual/src/main/asciidoc/past-event-conditions.adoc index d038f263b2..f2c3f1601a 100644 --- a/manual/src/main/asciidoc/past-event-conditions.adoc +++ b/manual/src/main/asciidoc/past-event-conditions.adoc @@ -31,9 +31,6 @@ Past event conditions are a powerful feature in Apache Unomi that allows both se [plantuml] ---- @startuml -skinparam activityBackgroundColor LightBlue -skinparam activityBorderColor DarkBlue -skinparam arrowColor DarkBlue |Visitor| start @@ -122,14 +119,11 @@ Common website scenarios include: } ---- -=== Mobile Application Engagement +==== Mobile Application Engagement [plantuml] ---- @startuml -skinparam activityBackgroundColor LightGreen -skinparam activityBorderColor DarkGreen -skinparam arrowColor DarkGreen |User| start @@ -201,17 +195,12 @@ Common mobile scenarios include: } ---- -=== Customer Data Platform (CDP) Integration +==== Customer Data Platform (CDP) Integration [plantuml] ---- @startuml skinparam componentStyle uml2 -skinparam component { - BackgroundColor<> LightYellow - BackgroundColor<> LightBlue - BackgroundColor<> LightGreen -} package "Data Sources" { [Website Events] as web <> @@ -305,14 +294,11 @@ Common CDP scenarios include: } ---- -=== B2B Use Cases +==== B2B Use Cases [plantuml] ---- @startuml -skinparam activityBackgroundColor LightPurple -skinparam activityBorderColor DarkPurple -skinparam arrowColor DarkPurple |Account| start @@ -392,16 +378,12 @@ Common B2B scenarios include: } ---- -== Architecture Overview +==== Architecture Overview [plantuml] ---- @startuml skinparam componentStyle uml2 -skinparam component { - BackgroundColor<> LightGreen - BackgroundColor<> LightYellow -} package "Past Event System" { [SetEventOccurenceCountAction] as action @@ -444,7 +426,7 @@ end note @enduml ---- -=== Query Strategy Selection +===== Query Strategy Selection [plantuml] ---- @@ -472,7 +454,7 @@ stop @enduml ---- -==== Performance Implications +====== Performance Implications 1. *Property-Based Evaluation* - Uses cached counts from profile properties @@ -486,14 +468,11 @@ stop - No caching mechanism - Use with caution in rules -== Event Processing Flow +==== Event Processing Flow [plantuml] ---- @startuml -skinparam activityBackgroundColor LightBlue -skinparam activityBorderColor DarkBlue -skinparam arrowColor DarkBlue start :Event Received; @@ -520,16 +499,12 @@ stop @enduml ---- -== Segment vs Direct Rule Comparison +==== Segment vs Direct Rule Comparison [plantuml] ---- @startuml skinparam componentStyle uml2 -skinparam component { - BackgroundColor<> LightGreen - BackgroundColor<> LightPink -} package "Segment Approach" <> { [Past Event Condition] as segmentCondition @@ -570,7 +545,7 @@ end note @enduml ---- -== Cache Update Process +==== Cache Update Process [plantuml] ---- @@ -601,7 +576,7 @@ deactivate action @enduml ---- -== Query Strategy Selection +==== Query Strategy Selection [plantuml] ---- @@ -629,9 +604,9 @@ stop @enduml ---- -== Usage Patterns +==== Usage Patterns -=== In Segments +===== In Segments When used in segments, past event conditions automatically generate optimization rules that maintain cached event counts on profiles. This is the recommended approach for optimal performance. @@ -661,7 +636,7 @@ When this segment is created: 3. Initial profile counts are calculated and stored 4. Subsequent events update the cached counts in real-time -=== In Rules +===== In Rules Past event conditions can also be used directly in rules, but with important performance considerations: @@ -699,7 +674,7 @@ Past event conditions can also be used directly in rules, but with important per 4. Higher latency and resource usage 5. Not recommended for high-frequency conditions -== Auto-Generated Rules +==== Auto-Generated Rules The system automatically generates rules for past event conditions through `SegmentServiceImpl`: @@ -744,7 +719,7 @@ The system automatically generates rules for past event conditions through `Segm } ---- -== Core Components +==== Core Components The past event condition system consists of four main components: @@ -753,7 +728,7 @@ The past event condition system consists of four main components: 3. `pastEventConditionQueryBuilder` - Constructs optimized search index queries 4. `SegmentServiceImpl` - Manages segments and auto-generated rules -== Condition Parameters +==== Condition Parameters Past event conditions support the following parameters as defined in `pastEventCondition.json`: @@ -827,9 +802,9 @@ If you need to combine multiple event conditions, you have two options: This pattern works because the condition type system is hierarchical - a condition type can define a `parentCondition` that will be evaluated as part of the condition evaluation. The restriction on using `booleanCondition` applies only to direct usage in the `eventCondition` parameter, not to the internal structure of properly tagged condition types. ==== -== Event Processing System +==== Event Processing System -=== Real-Time Event Processing +===== Real-Time Event Processing When an event occurs in Unomi, the `SetEventOccurenceCountAction` processes it in real-time through the following steps: @@ -856,7 +831,7 @@ The action is defined in `setEventOccurenceCountAction.json` with the following } ---- -=== Time Window Processing +===== Time Window Processing The `SetEventOccurenceCountAction` implements time window processing using the following logic: @@ -887,7 +862,7 @@ private boolean inTimeRange(LocalDateTime eventTime, Integer numberOfDays, } ---- -=== Profile Storage Format +===== Profile Storage Format Past event counts are stored efficiently in the profile's system properties under a `pastEvents` list. Each entry contains: @@ -925,7 +900,7 @@ private boolean updatePastEvents(Event event, String generatedPropertyKey, long } ---- -=== Evaluation Process +===== Evaluation Process The evaluation process involves multiple components working together to efficiently evaluate past event conditions: @@ -1009,9 +984,9 @@ public boolean eval(Condition condition, Item item, Map context, } ---- -== Rule Generation +==== Rule Generation -=== Auto-Generated Rules +===== Auto-Generated Rules The system automatically generates rules for past event conditions through `SegmentServiceImpl`: @@ -1041,7 +1016,7 @@ The system automatically generates rules for past event conditions through `Segm } ---- -=== Performance Configuration +===== Performance Configuration Key configuration parameters from `SegmentServiceImpl`: @@ -1069,9 +1044,9 @@ secondsDelayForRetryUpdateProfileSegment = 1 sendProfileUpdateEventForSegmentUpdate = true ---- -== Best Practices +==== Best Practices -=== Implementation Guidelines +===== Implementation Guidelines 1. *Time Window Selection* * Use `numberOfDays` for rolling windows @@ -1091,7 +1066,7 @@ sendProfileUpdateEventForSegmentUpdate = true * Implement proper logging * Plan for recovery scenarios -=== Example Configurations +===== Example Configurations 1. *Rolling Window* [source,json] @@ -1128,9 +1103,9 @@ sendProfileUpdateEventForSegmentUpdate = true } ---- -== Common Use Cases and Examples +==== Common Use Cases and Examples -=== Basic Past Event Condition +===== Basic Past Event Condition .Example: Count page views in the last 7 days [source,json] @@ -1146,7 +1121,7 @@ sendProfileUpdateEventForSegmentUpdate = true } ---- -=== Time-Based Conditions +===== Time-Based Conditions .Example: Count purchases between specific dates [source,json] @@ -1162,7 +1137,7 @@ sendProfileUpdateEventForSegmentUpdate = true } ---- -=== Complex Event Conditions +===== Complex Event Conditions .Example: Count specific product category views with property constraints [source,json] @@ -1201,7 +1176,7 @@ sendProfileUpdateEventForSegmentUpdate = true } ---- -=== Segment Definition +===== Segment Definition .Example: Segment for active users based on past events [source,json] @@ -1239,7 +1214,7 @@ sendProfileUpdateEventForSegmentUpdate = true } ---- -=== Auto-Generated Rule +===== Auto-Generated Rule .Example: Auto-generated rule for past event counting [source,json] @@ -1270,7 +1245,7 @@ sendProfileUpdateEventForSegmentUpdate = true } ---- -=== Property Storage Format +===== Property Storage Format .Example: Profile property storage format for past event counts [source,json] @@ -1291,9 +1266,9 @@ sendProfileUpdateEventForSegmentUpdate = true } ---- -=== Integration Examples +===== Integration Examples -==== REST API Usage +====== REST API Usage .Example: Query past event counts via REST API [source,bash] @@ -1312,7 +1287,7 @@ curl -X POST http://localhost:8181/cxs/profiles/query \ }' ---- -==== Batch Update Configuration +====== Batch Update Configuration .Example: Configure batch updates for past event counts [source,properties] @@ -1327,22 +1302,22 @@ segment.pastEventCondition.monitoringEnabled=true segment.pastEventCondition.alertThreshold=100000 ---- -== Troubleshooting +==== Troubleshooting -=== Common Issues +===== Common Issues 1. *Performance Problems* * Partition Size Issues ** Symptom: Slow query performance or high memory usage ** Check `aggregateQueryBucketSize` configuration (default: 5000) - ** Monitor Elasticsearch heap usage during queries + ** Monitor search engine (Elasticsearch or OpenSearch) heap usage during queries ** Consider enabling partitioned processing for large datasets * Query Optimization ** Verify proper index settings for event timestamps ** Check if property-based queries are being used when possible ** Monitor query execution times through metrics - ** Analyze Elasticsearch query patterns + ** Analyze search engine query patterns 2. *Incorrect Counts* * Time Window Configuration @@ -1362,11 +1337,11 @@ segment.pastEventCondition.alertThreshold=100000 ** Monitor profile update events ** Validate segment evaluation timing -=== Debugging Tips +===== Debugging Tips 1. *Query Verification* - * Elasticsearch Query Analysis - ** Use Elasticsearch _explain API to analyze queries + * Search engine query analysis + ** Use the Elasticsearch or OpenSearch `_explain` API to analyze queries ** Monitor query performance through metrics ** Check query routing and shard distribution ** Verify index mappings for event fields @@ -1384,9 +1359,9 @@ segment.pastEventCondition.alertThreshold=100000 ** Verify rule execution flow ** Validate condition evaluation results -== Integration Points +==== Integration Points -=== Event Flow +===== Event Flow The complete event processing flow for past event conditions: @@ -1414,7 +1389,7 @@ The complete event processing flow for past event conditions: * Profile is updated with new segments * Related rules are triggered -=== Segment Evaluation Flow +===== Segment Evaluation Flow The detailed segment evaluation process: @@ -1442,9 +1417,9 @@ The detailed segment evaluation process: * Update events are triggered * Changes are persisted to storage -== Security Considerations +==== Security Considerations -=== Data Access Controls +===== Data Access Controls 1. *Profile Data Protection* * Event data access is controlled through permissions @@ -1453,12 +1428,12 @@ The detailed segment evaluation process: * Access to past event counts follows profile permissions 2. *Query Security* - * Elasticsearch queries are sanitized + * Search engine queries are sanitized * Input parameters are validated * Time ranges are bounded * Resource limits are enforced -=== Resource Protection +===== Resource Protection 1. *Query Limits* * Maximum time window restrictions @@ -1472,7 +1447,7 @@ The detailed segment evaluation process: * Segment evaluation throttling * Cache invalidation controls -=== Data Integrity +===== Data Integrity 1. *Event Data* * Event timestamps are validated @@ -1486,7 +1461,7 @@ The detailed segment evaluation process: * Update validation * Rollback procedures -=== Audit Trail +===== Audit Trail 1. *Event Processing* * Event processing logs @@ -1500,9 +1475,9 @@ The detailed segment evaluation process: * Resource limit violations * Data integrity issues -== Configuration Parameters +==== Configuration Parameters -=== Core Settings +===== Core Settings [source,properties] ---- @@ -1521,7 +1496,7 @@ query.pastEvent.enablePartitioning=true query.pastEvent.partitionSize=10000 ---- -=== Performance Tuning +===== Performance Tuning [source,properties] ---- @@ -1539,7 +1514,7 @@ batch.pastEventUpdate.threadPoolSize=4 batch.pastEventUpdate.queueSize=1000 ---- -=== Monitoring Configuration +===== Monitoring Configuration [source,properties] ---- @@ -1556,9 +1531,9 @@ alerts.pastEventCondition.slowQueryThreshold=5000 alerts.pastEventCondition.errorThreshold=100 ---- -== Best Practices +==== Best Practices -=== Performance Optimization +===== Performance Optimization 1. *Query Optimization* * Use property-based evaluation when possible @@ -1572,7 +1547,7 @@ alerts.pastEventCondition.errorThreshold=100 * Monitor memory usage * Use partitioned processing -=== Maintenance +===== Maintenance 1. *Regular Tasks* * Monitor cache hit rates @@ -1586,7 +1561,7 @@ alerts.pastEventCondition.errorThreshold=100 * Review audit trails * Check resource usage -=== Scaling Considerations +===== Scaling Considerations 1. *Horizontal Scaling* * Configure cluster settings @@ -1600,11 +1575,11 @@ alerts.pastEventCondition.errorThreshold=100 * Optimize thread pools * Monitor CPU usage -== Evaluation Strategies +==== Evaluation Strategies The system uses different evaluation strategies depending on how the past event condition is used: -=== Property-Based Evaluation +===== Property-Based Evaluation Used when a past event condition is part of a segment: @@ -1633,12 +1608,12 @@ if (parameters.containsKey("generatedPropertyKey")) { } ---- -=== Direct Event Query Evaluation +===== Direct Event Query Evaluation Used when a past event condition is used directly in a rule: 1. *Evaluation Process* - * Constructs Elasticsearch query for matching events + * Constructs a search engine query for matching events * Queries event store directly * Aggregates results * No caching occurs @@ -1655,9 +1630,9 @@ count = persistenceService.queryCount( ); ---- -== Performance Considerations +==== Performance Considerations -=== Segment vs Direct Rule Usage +===== Segment vs Direct Rule Usage 1. *Segment Usage (Recommended)* * Cached event counts in profile properties @@ -1673,7 +1648,7 @@ count = persistenceService.queryCount( * Can impact system performance * Suitable only for low-frequency rules -=== Query Optimization +===== Query Optimization 1. *Property-Based Queries* * Used when `generatedPropertyKey` is available @@ -1710,7 +1685,7 @@ Set ids = getProfileIdsMatchingEventCount( ); ---- -=== Resource Usage +===== Resource Usage 1. *Memory Impact* * Segment usage: Minimal (only stores count in profile) @@ -1724,7 +1699,7 @@ Set ids = getProfileIdsMatchingEventCount( * Segment usage: Small profile property overhead * Direct rule usage: No additional storage -=== Best Practices for Performance +===== Best Practices for Performance 1. *Use Segments When Possible* * Prefer segments over direct rule usage @@ -1746,7 +1721,7 @@ Set ids = getProfileIdsMatchingEventCount( * Configure appropriate cache sizes * Regular cache maintenance -=== Resolution Steps +===== Resolution Steps 1. *Initial Analysis* * Verify configuration @@ -1772,9 +1747,9 @@ Set ids = getProfileIdsMatchingEventCount( * Check error rates * Document changes -== Maintenance and Monitoring +==== Maintenance and Monitoring -=== Regular Maintenance Tasks +===== Regular Maintenance Tasks 1. *Event Count Recalculation* * Periodic recalculation of past event counts @@ -1796,7 +1771,7 @@ segmentService.recalculatePastEventConditions(); * Monitor resource usage * Identify bottlenecks -=== Monitoring Metrics +===== Monitoring Metrics 1. *Query Performance* * Average query execution time @@ -1808,7 +1783,7 @@ segmentService.recalculatePastEventConditions(); * CPU utilization * Storage growth -=== Troubleshooting +===== Troubleshooting 1. *Common Issues* * Slow query performance @@ -1817,7 +1792,7 @@ segmentService.recalculatePastEventConditions(); * Cache inconsistencies 2. *Diagnostic Tools* - * Elasticsearch query analysis + * Search engine query analysis * Performance metrics * Log analysis * Cache statistics diff --git a/manual/src/main/asciidoc/plantuml/unomi-theme.puml b/manual/src/main/asciidoc/plantuml/unomi-theme.puml new file mode 100644 index 0000000000..7d7b86caba --- /dev/null +++ b/manual/src/main/asciidoc/plantuml/unomi-theme.puml @@ -0,0 +1,259 @@ +' +' Licensed under the Apache License, Version 2.0 (the "License"); +' you may not use this file except in compliance with the License. +' You may obtain a copy of the License at +' +' http://www.apache.org/licenses/LICENSE-2.0 +' +' Unless required by applicable law or agreed to in writing, software +' distributed under the License is distributed on an "AS IS" BASIS, +' WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +' See the License for the specific language governing permissions and +' limitations under the License. +' + +' Apache Unomi PlantUML theme — aligned with unomi-site / unomi-manual.css tokens +' Brand: --unomi-primary #6d5ce7, semantic accents from the Unomi design system +' +' Applied globally via AsciiDoctor attribute plantuml-config (see manual/pom.xml). +' Prefer stereotypes below for role colors; avoid per-diagram LightBlue/LightGreen overrides. + +' ---- Brand & semantic palette (match unomi.css :root) ---- +' primary #6d5ce7 | primary-dark #5a48c9 | primary-light #ede9fe | primary-subtle #f5f3ff +' text #1e293b | muted #64748b | border #e2e8f0 | bg-soft #f8fafc +' success #059669 / #ecfdf5 | info #2563eb / #dbeafe | amber #d97706 / #fef3c7 +' rose #db2777 / #fce7f3 | accent #00cec9 | hero-accent #a78bfa + +' ---- C4-PlantUML defaults (set before C4 include via ?= in C4) ---- +!$ELEMENT_FONT_COLOR = "#1e293b" +!$ARROW_COLOR = "#6d5ce7" +!$ARROW_FONT_COLOR = "#64748b" +!$BOUNDARY_COLOR = "#5a48c9" +!$BOUNDARY_BG_COLOR = "#f5f3ff" +!$PERSON_BG_COLOR = "#ede9fe" +!$PERSON_BORDER_COLOR = "#6d5ce7" +!$PERSON_FONT_COLOR = "#1e293b" +!$SYSTEM_BG_COLOR = "#ede9fe" +!$SYSTEM_BORDER_COLOR = "#6d5ce7" +!$SYSTEM_FONT_COLOR = "#1e293b" +!$CONTAINER_BG_COLOR = "#f5f3ff" +!$CONTAINER_BORDER_COLOR = "#6d5ce7" +!$CONTAINER_FONT_COLOR = "#1e293b" +!$COMPONENT_BG_COLOR = "#ede9fe" +!$COMPONENT_BORDER_COLOR = "#6d5ce7" +!$COMPONENT_FONT_COLOR = "#1e293b" +!$EXTERNAL_COMPONENT_BG_COLOR = "#f1f5f9" +!$EXTERNAL_COMPONENT_BORDER_COLOR = "#94a3b8" +!$EXTERNAL_COMPONENT_FONT_COLOR = "#1e293b" +!$DATABASE_BG_COLOR = "#ecfdf5" +!$DATABASE_BORDER_COLOR = "#059669" +!$DATABASE_FONT_COLOR = "#1e293b" + +skinparam { + shadowing false + defaultFontSize 13 + defaultFontColor #1e293b + BackgroundColor #ffffff + ArrowColor #6d5ce7 + ArrowFontColor #64748b + BorderColor #6d5ce7 + RoundCorner 8 + Padding 6 + NoteBackgroundColor #f5f3ff + NoteBorderColor #a78bfa + NoteFontColor #1e293b + PackageBackgroundColor #f8fafc + PackageBorderColor #e2e8f0 + PackageTitleFontColor #5a48c9 + StereotypeFontColor #64748b + LegendBackgroundColor #f8fafc + LegendBorderColor #e2e8f0 +} + +skinparam componentStyle uml2 + +skinparam activity { + BackgroundColor #ede9fe + BorderColor #6d5ce7 + FontColor #1e293b + DiamondBackgroundColor #f5f3ff + DiamondBorderColor #5a48c9 + StartColor #6d5ce7 + EndColor #5a48c9 + BarColor #a78bfa +} + +skinparam component { + BackgroundColor #ede9fe + BorderColor #6d5ce7 + FontColor #1e293b + ArrowColor #6d5ce7 +} + +skinparam class { + BackgroundColor #ffffff + BorderColor #6d5ce7 + FontColor #1e293b + AttributeFontColor #475569 + HeaderBackgroundColor #f5f3ff +} + +skinparam object { + BackgroundColor #ffffff + BorderColor #6d5ce7 + FontColor #1e293b +} + +skinparam rectangle { + BackgroundColor #ede9fe + BorderColor #6d5ce7 + FontColor #1e293b +} + +skinparam state { + BackgroundColor #ede9fe + BorderColor #6d5ce7 + FontColor #1e293b + StartColor #6d5ce7 + EndColor #5a48c9 +} + +skinparam sequence { + ArrowColor #6d5ce7 + LifeLineBorderColor #a78bfa + LifeLineBackgroundColor #f5f3ff + ParticipantBackgroundColor #ede9fe + ParticipantBorderColor #6d5ce7 + ParticipantFontColor #1e293b + ActorBackgroundColor #ede9fe + ActorBorderColor #6d5ce7 + BoxBackgroundColor #f8fafc + BoxBorderColor #e2e8f0 +} + +skinparam database { + BackgroundColor #ecfdf5 + BorderColor #059669 + FontColor #1e293b +} + +skinparam agent { + BackgroundColor #ede9fe + BorderColor #6d5ce7 + FontColor #1e293b +} + +skinparam node { + BackgroundColor #f5f3ff + BorderColor #6d5ce7 + FontColor #1e293b +} + +skinparam cloud { + BackgroundColor #e0f2fe + BorderColor #0369a1 + FontColor #1e293b +} + +skinparam interface { + BackgroundColor #f5f3ff + BorderColor #6d5ce7 + FontColor #1e293b +} + +' ---- Role stereotypes (shared across chapters) ---- +' primary / core path +skinparam component<> { + BackgroundColor #ede9fe + BorderColor #6d5ce7 +} +skinparam component<> { + BackgroundColor #ede9fe + BorderColor #5a48c9 +} +skinparam component<> { + BackgroundColor #ede9fe + BorderColor #6d5ce7 +} +skinparam component<> { + BackgroundColor #dbeafe + BorderColor #2563eb +} +skinparam component<> { + BackgroundColor #dbeafe + BorderColor #2563eb +} + +' persistence / query / success path +skinparam component<> { + BackgroundColor #ecfdf5 + BorderColor #059669 +} +skinparam component<> { + BackgroundColor #ecfdf5 + BorderColor #059669 +} +skinparam component<> { + BackgroundColor #ecfdf5 + BorderColor #059669 +} +skinparam component<> { + BackgroundColor #ecfdf5 + BorderColor #059669 +} +skinparam component<> { + BackgroundColor #ecfdf5 + BorderColor #059669 +} +skinparam component<> { + BackgroundColor #ecfdf5 + BorderColor #059669 +} +skinparam component<> { + BackgroundColor #ecfdf5 + BorderColor #059669 +} +skinparam component<> { + BackgroundColor #ecfdf5 + BorderColor #059669 +} + +' caution / security / pin +skinparam component<> { + BackgroundColor #fef3c7 + BorderColor #d97706 +} +skinparam component<> { + BackgroundColor #fef3c7 + BorderColor #d97706 +} +skinparam component<> { + BackgroundColor #fef3c7 + BorderColor #d97706 +} +skinparam component<> { + BackgroundColor #fef3c7 + BorderColor #d97706 +} + +' accent / embed +skinparam component<> { + BackgroundColor #e0f2fe + BorderColor #0369a1 +} + +' muted / external / terminal +skinparam component<> { + BackgroundColor #f1f5f9 + BorderColor #94a3b8 +} +skinparam state<> { + BackgroundColor #f1f5f9 + BorderColor #94a3b8 +} + +' negative / inefficient +skinparam component<> { + BackgroundColor #fce7f3 + BorderColor #db2777 +} diff --git a/manual/src/main/asciidoc/privacy.adoc b/manual/src/main/asciidoc/privacy.adoc index 3384ba6be9..dda5c3959b 100644 --- a/manual/src/main/asciidoc/privacy.adoc +++ b/manual/src/main/asciidoc/privacy.adoc @@ -60,10 +60,11 @@ IDs. [source] ---- curl -X GET http://localhost:8181/cxs/client/myprofile.[json,csv,yaml,text] \ +-H "X-Unomi-Api-Key: YOUR_PUBLIC_API_KEY" \ --cookie "context-profile-id=PROFILE-ID" ---- -where `PROFILE-ID` is the profile identifier for which to download the profile. +where `PROFILE-ID` is the profile identifier for which to download the profile, and `YOUR_PUBLIC_API_KEY` is the tenant public API key (required on Unomi 3.1 public `/cxs/client/*` endpoints unless <<_v2_compatibility_mode,V2 compatibility mode>> is enabled). === Deleting a profile diff --git a/manual/src/main/asciidoc/property-types.adoc b/manual/src/main/asciidoc/property-types.adoc index dec203ad36..7d83164331 100644 --- a/manual/src/main/asciidoc/property-types.adoc +++ b/manual/src/main/asciidoc/property-types.adoc @@ -13,13 +13,13 @@ // [#_property_types] -== Property Types +=== Property Types Property types define the structure and metadata for properties that can be used in profiles and sessions within Apache Unomi. They specify the data type, display hints (ranges, constraints), default values, and other metadata that help inform UIs and enable rich querying capabilities. IMPORTANT: Property types are primarily designed for **UI purposes** - they inform user interfaces about what properties should be editable, how they should be displayed, and what constraints apply. Property types are **dynamically created and updated** at runtime and do not enforce strict validation on the server side. For actual data validation, Apache Unomi uses <<_json_schemas,JSON Schemas>>, which are used exclusively for validating events sent through public endpoints. -=== Quick Reference: Property Types vs JSON Schemas +==== Quick Reference: Property Types vs JSON Schemas |=== | Aspect | Property Types | JSON Schemas @@ -40,7 +40,7 @@ IMPORTANT: Property types are primarily designed for **UI purposes** - they info |=== -=== Overview +==== Overview Property types serve as **informational schemas** for properties, primarily used by UIs to: @@ -53,7 +53,7 @@ Property types serve as **informational schemas** for properties, primarily used * Whether the property is protected (read-only) * Display metadata (names, descriptions, tags, ranks) for UI organization -=== Property Type Targets +==== Property Type Targets Property types are organized by their `target`, which indicates which type of object the property applies to: @@ -67,11 +67,11 @@ The target is typically determined automatically from the file path when propert NOTE: Property types are only used for profiles and sessions. Events are immutable records and should not be edited through property type-based UIs. Event validation is handled by JSON schemas, not property types. -=== Property Type Structure +==== Property Type Structure A property type is defined using the following structure: -==== Structure Definition +===== Structure Definition Inherits all the fields from: <> @@ -104,7 +104,7 @@ Inherits all the fields from: <> |=== -==== Metadata Fields +===== Metadata Fields Property types inherit from MetadataItem, which provides: @@ -131,7 +131,7 @@ Property types inherit from MetadataItem, which provides: |=== -=== Value Types +==== Value Types Property types use value types to define the primitive data type. Common value types include: @@ -145,9 +145,9 @@ Property types use value types to define the primitive data type. Common value t * `email` - Email addresses * `geoPoint` - Geographic coordinates -=== Examples +==== Examples -==== Profile Property Type Example +===== Profile Property Type Example This example shows a profile property type for storing a person's age with numeric ranges: @@ -178,7 +178,7 @@ This example shows a profile property type for storing a person's age with numer } ---- -==== Session Property Type Example +===== Session Property Type Example This example shows a session property type for storing geographic information: @@ -201,7 +201,7 @@ This example shows a session property type for storing geographic information: } ---- -==== Simple Profile Property Type Example +===== Simple Profile Property Type Example This example shows a basic profile property type for storing a first name: @@ -227,11 +227,11 @@ This example shows a basic profile property type for storing a first name: } ---- -=== Dynamic Creation and Management +==== Dynamic Creation and Management Property types are designed to be **dynamically created and updated at runtime**. This makes them ideal for scenarios where property definitions need to change without code deployments or server restarts. -==== Key Characteristics +===== Key Characteristics * **Runtime Creation**: Property types can be created, updated, and deleted via REST API or GraphQL without restarting the server * **No Code Deployment Required**: Changes to property types take effect immediately @@ -239,7 +239,7 @@ Property types are designed to be **dynamically created and updated at runtime** * **Flexible**: Can be modified to adapt to changing business requirements * **Multi-tenant**: Property types can be scoped to specific tenants -==== Use Cases for Dynamic Property Types +===== Use Cases for Dynamic Property Types * **Custom Fields**: Allow users to define custom profile or session properties through a UI * **A/B Testing**: Quickly add or modify property definitions for testing @@ -247,11 +247,11 @@ Property types are designed to be **dynamically created and updated at runtime** * **Integration Flexibility**: Add properties for new integrations without code changes * **Business Rule Changes**: Update property constraints and metadata as business rules evolve -=== Creating Property Types +==== Creating Property Types Property types can be created in several ways: -==== Via JSON Files in Plugins +===== Via JSON Files in Plugins The most common way is to include property type definitions as JSON files in your plugin bundle: @@ -265,7 +265,7 @@ The most common way is to include property type definitions as JSON files in you 3. Subdirectories can be used for organization (e.g., `profiles/personal/`, `profiles/contact/`) -==== Via REST API +===== Via REST API Property types can be created or updated using the REST API: @@ -287,13 +287,13 @@ Content-Type: application/json } ---- -==== Via GraphQL API +===== Via GraphQL API Property types can also be managed through the GraphQL API using property type mutations and queries. -=== Querying Property Types +==== Querying Property Types -==== Get All Property Types by Target +===== Get All Property Types by Target Retrieve all property types for a specific target: @@ -319,7 +319,7 @@ Response: ] ---- -==== Get All Property Types +===== Get All Property Types Retrieve all property types grouped by target: @@ -338,7 +338,7 @@ Response: } ---- -==== Get Property Type by ID +===== Get Property Type by ID Retrieve a specific property type: @@ -347,7 +347,7 @@ Retrieve a specific property type: GET /cxs/profiles/properties/firstName ---- -==== Get Property Types by Tag +===== Get Property Types by Tag Retrieve property types with specific tags: @@ -356,7 +356,7 @@ Retrieve property types with specific tags: GET /cxs/profiles/properties/tags/profileProperties,personalProfileProperties ---- -==== Get Property Types by System Tag +===== Get Property Types by System Tag Retrieve property types with specific system tags: @@ -365,9 +365,9 @@ Retrieve property types with specific system tags: GET /cxs/profiles/properties/systemTags/profileProperties ---- -=== Property Type Features +==== Property Type Features -==== Multi-valued Properties +===== Multi-valued Properties Set `multivalued` to `true` to allow a property to contain multiple values: @@ -384,7 +384,7 @@ Set `multivalued` to `true` to allow a property to contain multiple values: } ---- -==== Protected Properties +===== Protected Properties Set `protected` to `true` to make a property read-only (note: use `"protected"` in JSON, not `"protekted"`): @@ -401,7 +401,7 @@ Set `protected` to `true` to make a property read-only (note: use `"protected"` } ---- -==== Numeric Ranges +===== Numeric Ranges Define ranges for numeric properties to enable categorization: @@ -421,7 +421,7 @@ Define ranges for numeric properties to enable categorization: } ---- -==== Date Ranges +===== Date Ranges Define ranges for date properties: @@ -440,7 +440,7 @@ Define ranges for date properties: } ---- -==== Merge Strategies +===== Merge Strategies Specify how properties should be merged when profiles are combined: @@ -466,7 +466,7 @@ Common merge strategies include: NOTE: The merge strategy identifier in property type JSON files should match the `id` field from the merge strategy definition JSON files located in the `META-INF/cxs/mergers/` directory (this is a file system path within plugin bundles, not an API endpoint). -==== Nested Properties +===== Nested Properties For complex objects, use `childPropertyTypes` to define nested structures: @@ -504,11 +504,11 @@ For complex objects, use `childPropertyTypes` to define nested structures: } ---- -=== Tags and System Tags +==== Tags and System Tags Property types support two types of tags for categorization: `tags` and `systemTags`. Understanding the difference between them is important for organizing and filtering property types in UIs. -==== Tags vs System Tags +===== Tags vs System Tags **Tags** (`tags` field): * User-editable tags that can be modified through UIs @@ -524,17 +524,17 @@ Property types support two types of tags for categorization: `tags` and `systemT * Help classify properties for internal system logic * Example: `profileProperties`, `sessionProperties`, `personalIdentifierProperties` -==== Using Tags to Group Properties in UIs +===== Using Tags to Group Properties in UIs Tags provide a flexible way to organize property types for display and editing in user interfaces. Here's how you can leverage tags: -===== UI Grouping Strategy +====== UI Grouping Strategy 1. **Create Custom Tags**: Assign custom tags to property types based on how you want to group them in your UI 2. **Query by Tags**: Use the REST API to retrieve property types filtered by specific tags 3. **Display in Groups**: Organize properties in tabs, sections, or accordions based on their tags -===== Example: Grouping Properties by Department +====== Example: Grouping Properties by Department Suppose you want to group profile properties by department in your UI: @@ -585,7 +585,7 @@ GET /cxs/profiles/properties/tags/sales GET /cxs/profiles/properties/tags/support ---- -===== Example: Multi-Tag Filtering +====== Example: Multi-Tag Filtering Properties can have multiple tags, allowing for flexible filtering: @@ -609,13 +609,13 @@ You can query by multiple tags (comma-separated): GET /cxs/profiles/properties/tags/customer-tier,priority,billing ---- -==== Creating and Assigning Tags +===== Creating and Assigning Tags -===== Creating Tags +====== Creating Tags Tags don't need to be pre-registered. Simply assign a tag string to a property type, and it becomes available for filtering. Tags are created implicitly when first used. -===== Assigning Tags via JSON Files +====== Assigning Tags via JSON Files When defining property types in JSON files, include tags in the metadata: @@ -633,7 +633,7 @@ When defining property types in JSON files, include tags in the metadata: } ---- -===== Assigning Tags via REST API +====== Assigning Tags via REST API You can add or modify tags when creating or updating a property type: @@ -654,7 +654,7 @@ Content-Type: application/json } ---- -===== Updating Tags on Existing Property Types +====== Updating Tags on Existing Property Types To add tags to an existing property type, retrieve it, modify the tags, and save it: @@ -678,13 +678,13 @@ Content-Type: application/json } ---- -===== Assigning Tags via GraphQL +====== Assigning Tags via GraphQL Tags can also be managed through the GraphQL API when creating or updating property types. -==== Tag-Based UI Organization Examples +===== Tag-Based UI Organization Examples -===== Example 1: Tabbed Interface +====== Example 1: Tabbed Interface Organize properties into tabs based on tags: @@ -708,7 +708,7 @@ console.log(' Tab: Support'); console.log(' PropertyEditor with', supportProps.length, 'properties'); ---- -===== Example 2: Accordion Sections +====== Example 2: Accordion Sections Group properties in collapsible sections: @@ -742,7 +742,7 @@ for (const [tag, properties] of Object.entries(grouped)) { } ---- -===== Example 3: Filterable List +====== Example 3: Filterable List Allow users to filter properties by tag: @@ -784,7 +784,7 @@ for (const tag of uniqueTags) { console.log(`PropertyList with ${filteredProps.length} properties`); ---- -==== Best Practices for Tags +===== Best Practices for Tags 1. **Use Descriptive Tag Names**: Choose clear, meaningful tag names (e.g., `marketing`, `sales`, `support` rather than `m`, `s`, `sup`) @@ -800,7 +800,7 @@ console.log(`PropertyList with ${filteredProps.length} properties`); 7. **Use Tags for Permissions**: Consider using tags to control which properties are visible or editable by different user roles -=== System Tags +==== System Tags System tags are used to categorize property types. Common system tags include: @@ -818,25 +818,25 @@ System tags are used to categorize property types. Common system tags include: * `geographicSessionProperties` - Geographic session information * `technicalSessionProperties` - Technical session information -=== Validation and Property Types +==== Validation and Property Types IMPORTANT: Property types provide **informational metadata** for UIs but do **not enforce server-side validation**. The ranges, types, and constraints defined in property types are hints for UI components, not strict validation rules. -==== What Property Types Do +===== What Property Types Do * **Inform UIs**: Tell user interfaces what properties exist, their types, and how to display them * **Provide Hints**: Give UI components information about expected data formats and constraints * **Enable Dynamic Forms**: Allow UIs to generate forms and property editors automatically * **Organize Display**: Help organize properties in UIs using tags, ranks, and categories -==== What Property Types Don't Do +===== What Property Types Don't Do * **Server-Side Validation**: Property types do not validate data on the server * **Enforce Constraints**: Ranges and constraints are informational, not enforced * **Prevent Invalid Data**: Invalid data can still be stored if not validated elsewhere * **Type Safety**: Property types don't guarantee type safety at the storage level -==== When Validation Happens +===== When Validation Happens * **JSON Schemas**: Used exclusively for server-side validation of events sent through `/context.json` and `/eventcollector` endpoints * **Property Types**: Provide no server-side validation - they are informational only @@ -844,7 +844,7 @@ IMPORTANT: Property types provide **informational metadata** for UIs but do **no * **Application Logic**: Add validation in your application code if needed for profiles and sessions * **Database Constraints**: Use database-level constraints if your persistence layer supports them -==== Best Practice: Property Types for Profiles/Sessions, JSON Schemas for Events +===== Best Practice: Property Types for Profiles/Sessions, JSON Schemas for Events Property types and JSON schemas serve different purposes for different objects: @@ -861,10 +861,6 @@ Property types and JSON schemas serve different purposes for different objects: [plantuml] ---- @startuml -skinparam rectangle { - BackgroundColor #E8F4F8 - BorderColor #4A90A4 -} rectangle "Profiles/Sessions Workflow" { rectangle "Property Type\n(UI hints)" as PT @@ -888,7 +884,7 @@ rectangle "Events Workflow" { @enduml ---- -=== Best Practices +==== Best Practices 1. **Use descriptive names**: Property type IDs should be clear and descriptive (e.g., `firstName` rather than `fn`). @@ -906,13 +902,13 @@ rectangle "Events Workflow" { 8. **Use appropriate value types**: Choose the correct value type (string, integer, date, etc.) for your data. -=== Built-in Property Types +==== Built-in Property Types Apache Unomi comes with a comprehensive set of built-in property types that cover common use cases. These are organized by category and target type. -==== Built-in Profile Property Types +===== Built-in Profile Property Types -===== Basic Profile Properties +====== Basic Profile Properties These properties store basic identifying information about profiles: @@ -923,7 +919,7 @@ These properties store basic identifying information about profiles: All basic profile properties are tagged with `basicProfileProperties`. `firstName` and `lastName` are also tagged with `personalIdentifierProperties`. -===== Personal Profile Properties +====== Personal Profile Properties These properties store personal information: @@ -934,7 +930,7 @@ These properties store personal information: All personal profile properties are tagged with `personalProfileProperties`. -===== Contact Profile Properties +====== Contact Profile Properties These properties store contact information: @@ -947,7 +943,7 @@ These properties store contact information: All contact profile properties are tagged with `contactProfileProperties`. Email, address, and phoneNumber are also tagged with `personalIdentifierProperties`. -===== Work Profile Properties +====== Work Profile Properties These properties store work-related information: @@ -957,7 +953,7 @@ These properties store work-related information: All work profile properties are tagged with `workProfileProperties`. -===== Social Profile Properties +====== Social Profile Properties These properties store social media identifiers: @@ -968,7 +964,7 @@ These properties store social media identifiers: All social profile properties are tagged with `socialProfileProperties`. -===== System Profile Properties +====== System Profile Properties These properties are automatically managed by Unomi and are typically protected (read-only): @@ -980,15 +976,15 @@ These properties are automatically managed by Unomi and are typically protected All system profile properties are tagged with `systemProfileProperties`. -===== Lead Profile Properties +====== Lead Profile Properties * `leadAssignedTo` (string) - Person assigned to the lead All lead profile properties are tagged with `leadProfileProperties`. -==== Built-in Session Property Types +===== Built-in Session Property Types -===== Geographic Session Properties +====== Geographic Session Properties These properties store geographic information about the session: @@ -1002,7 +998,7 @@ These properties store geographic information about the session: All geographic session properties are tagged with `geographicSessionProperties`. -===== Technical Session Properties +====== Technical Session Properties These properties store technical information about the session: @@ -1015,7 +1011,7 @@ These properties store technical information about the session: All technical session properties are tagged with `technicalSessionProperties`. -==== Property Type Organization +===== Property Type Organization Built-in property types are organized in the following directory structure: @@ -1037,7 +1033,7 @@ META-INF/cxs/properties/ The directory structure helps organize property types, and the target is automatically determined from the parent directory (`profiles` or `sessions`). -==== Viewing Built-in Property Types +===== Viewing Built-in Property Types You can view all built-in property types using the REST API: @@ -1054,11 +1050,11 @@ GET /cxs/profiles/properties/systemTags/profileProperties GET /cxs/profiles/properties/systemTags/sessionProperties ---- -=== Property Types vs JSON Schemas +==== Property Types vs JSON Schemas Property types and JSON schemas serve complementary but distinct roles in Apache Unomi: -==== Property Types: UI-Focused Metadata +===== Property Types: UI-Focused Metadata Property types are: * **Primarily for UIs**: Inform user interfaces about property structure, display names, types, and constraints @@ -1074,7 +1070,7 @@ Use property types when you need to: * Control property visibility and editability in interfaces * Provide hints about data types and constraints to UI components -==== JSON Schemas: Server-Side Validation +===== JSON Schemas: Server-Side Validation JSON schemas are: * **For validation**: Enforce strict data validation on the server side @@ -1091,17 +1087,13 @@ Use JSON schemas when you need to: * Enforce business rules and constraints on events * Validate event structures before processing -==== How They Work Together +===== How They Work Together Property types and JSON schemas complement each other: [plantuml] ---- @startuml -skinparam rectangle { - BackgroundColor #E8F4F8 - BorderColor #4A90A4 -} rectangle "Property Types" as PT note right of PT @@ -1141,7 +1133,7 @@ UI --> JSON : Data Submission 4. **Data Storage**: Data is stored in the profile (property types don't validate, but UI can implement client-side validation) 5. **Event Validation (separate)**: When events are sent through `/context.json` or `/eventcollector`, JSON schemas validate the event structure (not profile properties) -==== When to Use Each +===== When to Use Each **Use Property Types for:** * Building dynamic property editors @@ -1160,7 +1152,7 @@ UI --> JSON : Data Submission * You want UI hints for profiles/sessions (property types) AND need to validate events (JSON schemas) * Building UIs that display profile/session data (property types) while also handling event validation (JSON schemas) -=== Property Types for Events +==== Property Types for Events Property types are **not used for events**. Events are immutable records of user interactions and should not be edited through property type-based UIs. @@ -1171,11 +1163,11 @@ If you need to understand event structure for display purposes: Event validation is performed by JSON schemas (with target `"events"`) when events are sent through the public endpoints (`/context.json` and `/eventcollector`). -=== Building UIs with Property Types +==== Building UIs with Property Types Property types are designed to enable dynamic UI generation. Here's how to use them effectively: -==== Step 1: Fetch Property Types +===== Step 1: Fetch Property Types Retrieve property types for your target (profiles or sessions): @@ -1204,7 +1196,7 @@ const marketingProps: PropertyType[] = await fetch('/cxs/profiles/properties/tag .then(res => res.json()); ---- -==== Step 2: Generate Form Fields +===== Step 2: Generate Form Fields Use property type metadata to generate appropriate form fields: @@ -1262,7 +1254,7 @@ function generateFormField(propertyType: PropertyType): FormField { const formFields: FormField[] = propertyTypes.map(generateFormField); ---- -==== Step 3: Render Dynamic Forms +===== Step 3: Render Dynamic Forms Use the generated fields to render your form: @@ -1323,7 +1315,7 @@ function renderPropertyForm({ propertyTypes, onSubmit }: PropertyFormProps): voi } ---- -==== Step 4: Handle Numeric Ranges +===== Step 4: Handle Numeric Ranges For properties with numeric ranges, you can provide range-based inputs: @@ -1370,7 +1362,7 @@ function formatRange(range: NumericRange): string { } ---- -==== Step 5: Sort by Rank +===== Step 5: Sort by Rank Use the `rank` field to control display order: @@ -1388,7 +1380,7 @@ function sortByRank(propertyTypes: PropertyType[]): PropertyType[] { const sortedPropertyTypes: PropertyType[] = sortByRank(propertyTypes); ---- -==== Complete Example: Dynamic Property Editor +===== Complete Example: Dynamic Property Editor Here's a complete example of a dynamic property editor: @@ -1545,7 +1537,7 @@ function getInputType(valueType: string): string { } ---- -=== Related Topics +==== Related Topics * <<_data_model_overview,Data Model Overview>> - Learn about the overall data model * <<_writing_plugins,Writing Plugins>> - Learn how to create plugins with property types diff --git a/manual/src/main/asciidoc/queries-and-aggregations.adoc b/manual/src/main/asciidoc/queries-and-aggregations.adoc index 2f028f257f..d2fe43fbc6 100644 --- a/manual/src/main/asciidoc/queries-and-aggregations.adoc +++ b/manual/src/main/asciidoc/queries-and-aggregations.adoc @@ -22,7 +22,7 @@ In this section we will show examples of requests that may be built using this A Query counts are highly optimized queries that will count the number of objects that match a certain condition without retrieving the results. This can be used for example to quickly figure out how many objects will match a given condition -before actually retrieving the results. It uses search engine optimizations (ElasticSearch/OpenSearch) to avoid the cost of loading all the +before actually retrieving the results. It uses search engine optimizations (Elasticsearch/OpenSearch) to avoid the cost of loading all the resulting objects. Here's an example of a query: @@ -74,7 +74,7 @@ Metric queries make it possible to apply functions to the resulting property. Th - min - max -These metrics are supported by both ElasticSearch and OpenSearch backends. +These metrics are supported by both Elasticsearch and OpenSearch backends. It is also possible to request more than one metric in a single request by concatenating them with a "/" in the URL. Here's an example request that uses the `sum` and `avg` metrics: @@ -131,9 +131,10 @@ Aggregations may be of different types. They are listed here below. ===== Date -Date aggregations make it possible to automatically generate "buckets" by time periods. The format is compatible with both ElasticSearch and OpenSearch. +Date aggregations make it possible to automatically generate "buckets" by time periods. The format is compatible with both Elasticsearch and OpenSearch. For more information about the format, you can refer to: -- ElasticSearch documentation: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/search-aggregations-bucket-datehistogram-aggregation.html +- Elasticsearch documentation: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-datehistogram-aggregation.html[Date histogram aggregation (Elasticsearch current)] +- OpenSearch documentation: https://docs.opensearch.org/docs/latest/aggregations/bucket/date-histogram/[Date histogram aggregation (OpenSearch)] - OpenSearch documentation: https://opensearch.org/docs/latest/aggregations/bucket/datehistogram/ Here's an example of a request to retrieve a histogram of by day of all the session that have been create by newcomers (nbOfVisits=1) @@ -273,7 +274,8 @@ The resulting JSON response will look something like this: ---- You can find more information about the date range formats here: -- ElasticSearch documentation: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/search-aggregations-bucket-daterange-aggregation.html +- Elasticsearch documentation: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-daterange-aggregation.html[Date range aggregation (Elasticsearch current)] +- OpenSearch documentation: https://docs.opensearch.org/docs/latest/aggregations/bucket/date-range/[Date range aggregation (OpenSearch)] - OpenSearch documentation: https://opensearch.org/docs/latest/aggregations/bucket/daterange/ diff --git a/manual/src/main/asciidoc/request-examples.adoc b/manual/src/main/asciidoc/request-examples.adoc index e8a8e64155..cb06a6f6ba 100644 --- a/manual/src/main/asciidoc/request-examples.adoc +++ b/manual/src/main/asciidoc/request-examples.adoc @@ -36,33 +36,45 @@ curl -X POST http://localhost:8181/cxs/tenants \ }' ---- -The response will include the created tenant with automatically generated API keys: +The response includes the created tenant with **masked** API key metadata only (plaintext secrets are not returned): [source,json] ---- { "itemId": "mytenant", - "name": "My Company", - "description": "My tenant description", + "itemType": "tenant", + "status": "ACTIVE", "apiKeys": [ { - "type": "PUBLIC", - "key": "public-key-abc123...", - "created": "2024-01-01T00:00:00Z" + "maskedKey": "unomi_v1_****ab12", + "keyType": "PUBLIC", + "revoked": false }, { - "type": "PRIVATE", - "key": "private-key-xyz789...", - "created": "2024-01-01T00:00:00Z" + "maskedKey": "unomi_v1_****cd34", + "keyType": "PRIVATE", + "revoked": false } - ] + ], + "properties": { + "name": "My Company", + "description": "My tenant description" + } } ---- -After creating the tenant, you will need to use these credentials in the examples: +Regenerate keys to obtain one-time plaintext values (store them immediately): + +[source,bash] +---- +curl -X POST "http://localhost:8181/cxs/tenants/mytenant/apikeys?type=PUBLIC" --user karaf:karaf +curl -X POST "http://localhost:8181/cxs/tenants/mytenant/apikeys?type=PRIVATE" --user karaf:karaf +---- + +After creating the tenant and regenerating keys, use these credentials in the examples: - Tenant ID: `mytenant` -- Private API Key: Extract the `key` value from the API key with `type: "PRIVATE"` in the response -- Public API Key: Extract the `key` value from the API key with `type: "PUBLIC"` in the response +- Private API Key: `plainTextKey` from the PRIVATE key-creation response +- Public API Key: `plainTextKey` from the PUBLIC key-creation response ===== 2. Create a scope @@ -82,7 +94,7 @@ curl -X POST http://localhost:8181/cxs/scopes \ }' ---- -TIP: The scope creation response will include a public API key that you should save for use with the public APIs. +TIP: After creating the scope, continue using the tenant **public** API key (from `POST /cxs/tenants/{id}/apikeys?type=PUBLIC`) for `/cxs/context.json` and `/cxs/eventcollector`. Scope creation does not return an API key. [IMPORTANT] ==== @@ -101,7 +113,7 @@ NOTE: In all the examples below, replace: - `YOUR_TENANT_ID` with `mytenant` - `YOUR_PRIVATE_API_KEY` with your actual private API key - `example` scope with `mydigital` -- `YOUR_PUBLIC_API_KEY` with the public key from the scope creation response +- `YOUR_PUBLIC_API_KEY` with the tenant public API key from key regeneration ===== 3. Verify the scope diff --git a/manual/src/main/asciidoc/samples/login-sample.adoc b/manual/src/main/asciidoc/samples/login-sample.adoc index 30b7f1aee0..0a33e23771 100644 --- a/manual/src/main/asciidoc/samples/login-sample.adoc +++ b/manual/src/main/asciidoc/samples/login-sample.adoc @@ -14,7 +14,7 @@ [#_login_sample] === Login sample -This samples is an example of what is involved in integrated a login with Apache Unomi. +This sample is an example of what is involved in integrating a login with Apache Unomi. ==== Warning ! diff --git a/manual/src/main/asciidoc/samples/samples.adoc b/manual/src/main/asciidoc/samples/samples.adoc index a7f0db0b7f..8cdd480ae7 100644 --- a/manual/src/main/asciidoc/samples/samples.adoc +++ b/manual/src/main/asciidoc/samples/samples.adoc @@ -20,3 +20,4 @@ These are scenario walkthroughs, not maintained plugin templates — for plugin * <<_twitter_sample,Twitter integration>> * <<_login_sample,Login integration>> +* <<_weather_update_sample,Weather update extension>> diff --git a/manual/src/main/asciidoc/samples/twitter-sample.adoc b/manual/src/main/asciidoc/samples/twitter-sample.adoc index a45955d903..c03773eacc 100644 --- a/manual/src/main/asciidoc/samples/twitter-sample.adoc +++ b/manual/src/main/asciidoc/samples/twitter-sample.adoc @@ -43,7 +43,7 @@ If you are using the packaged version of Unomi (as opposed to deploying it to yo [source] ---- -cp target/tweet-button-plugin-2.0.0-SNAPSHOT.jar ../../package/target/unomi-2.0.0-SNAPSHOT/deploy +cp target/tweet-button-plugin-3.1.0-SNAPSHOT.jar ../../package/target/unomi-3.1.0-SNAPSHOT/deploy ---- ===== Testing the samples @@ -185,7 +185,7 @@ Let's look at the context request structure: "requiredProfileProperties": , "requiredSessionProperties": , filters: , - "personalitations": , + "personalizations": , "profileOverrides": , - segments: , - profileProperties: , @@ -615,4 +615,79 @@ We have seen a simple example how to interact with Unomi using a combination of Here is an overview of how Unomi processes incoming requests to the `ContextServlet`. -image::unomi-request.png[Unomi request overview] +[plantuml] +---- +@startuml +skinparam shadowing false +hide footbox + +participant ContextServlet +participant ProfileService +participant EventService +participant RulesService +participant PersistenceService + +[-> ContextServlet : Request + +== User identification == +ContextServlet -> ProfileService : Find/create user and session +ProfileService -> PersistenceService : Find/create user and session +PersistenceService --> ProfileService +ProfileService --> ContextServlet + +== Handle events == +loop all events + ContextServlet -> EventService : Send events + group Event handling + EventService -> RulesService : Call listener + RulesService -> RulesService : Get matching rules + loop all rules + RulesService -> PersistenceService : Test rule against current\nevent / source / profile / session + PersistenceService --> RulesService + loop all actions + RulesService -> RulesService : Execute action + end + RulesService -> EventService : Send 'rule fired' Event + note right of EventService + Recurse Event Handling + end note + end + RulesService -> EventService : Send profile updated event + note right of EventService + Recurse Event Handling + end note + end +end + +== Test condition filters == +loop all condition filters + ContextServlet -> ProfileService : Check condition against\ncurrent profile/session + ProfileService -> PersistenceService : Test condition against\ncurrent profile/session + PersistenceService --> ProfileService + ProfileService --> ContextServlet + note right of ContextServlet + Add filter results to answer + end note +end + +== Tracked Conditions == +ContextServlet -> RulesService : Get tracked conditions +loop all rules + RulesService -> PersistenceService : Test condition against\ncurrent event source + PersistenceService --> RulesService +end +RulesService --> ContextServlet +note right of ContextServlet + Add tracked conditions to answer +end note + +== Finalize == +ContextServlet -> ProfileService : Save profile and/or\nsession if needed +ProfileService -> PersistenceService : Save profile to persistence +PersistenceService --> ProfileService +ProfileService --> ContextServlet + +ContextServlet ->] : Response + +@enduml +---- diff --git a/manual/src/main/asciidoc/samples/weather-update-sample.adoc b/manual/src/main/asciidoc/samples/weather-update-sample.adoc index d40881ebbd..d4216bcdbf 100644 --- a/manual/src/main/asciidoc/samples/weather-update-sample.adoc +++ b/manual/src/main/asciidoc/samples/weather-update-sample.adoc @@ -11,4 +11,45 @@ // See the License for the specific language governing permissions and // limitations under the License. // -=== Weather update sample \ No newline at end of file +[#_weather_update_sample] +=== Weather update sample + +The **weather-update** extension enriches the visitor session with weather properties based on the resolved geo-location (typically from IP lookup). It calls the OpenWeatherMap API and stores values such as temperature, “feels like”, wind speed, and wind direction on the session. + +Source and packaging live under `extensions/weather-update/` in the Unomi repository. See also the extension README: https://github.com/apache/unomi/blob/master/extensions/weather-update/README.md + +==== Prerequisites + +* Apache Unomi running with a tenant and API keys (see <<_multitenancy,Multi-tenancy>>) +* An OpenWeatherMap account and API key: https://home.openweathermap.org/api_keys +* GeoIP / location resolution configured so sessions have a usable location (see Geonames / GeoIP sections in <<_configuration,Configuration>>) + +==== Install the extension + +From the Karaf shell (adjust the version to match your Unomi build): + +[source,bash] +---- +feature:repo-add mvn:org.apache.unomi/unomi-weather-update-karaf-kar/3.1.0-SNAPSHOT/xml/features +feature:install unomi-weather-update-karaf-kar +---- + +==== Configure the OpenWeatherMap API key + +Edit `etc/org.apache.unomi.weatherUpdate.cfg`: + +[source,properties] +---- +weatherUpdate.apiKey=YOUR_OPENWEATHERMAP_API_KEY +---- + +==== What it updates + +The extension registers session property types under the weather tag (for example `weatherTemp`, `weatherLike`, `weatherWindSpeed`, `weatherWindDirection`) and an action that fills them when a suitable event/rule fires. + +Wire a rule that runs the weather-update action on session start or first view (using the tenant private API key or JAAS), then verify session properties via `POST /cxs/profiles/search` / session APIs or a context request with `requiredSessionProperties`. + +==== Related + +* <<_twitter_sample,Twitter sample>> — another extension-driven action example +* <<_builtin_action_types,Built-in action types>> — action plugin patterns diff --git a/manual/src/main/asciidoc/security.adoc b/manual/src/main/asciidoc/security.adoc index a46996c62d..acfdd1b1fe 100644 --- a/manual/src/main/asciidoc/security.adoc +++ b/manual/src/main/asciidoc/security.adoc @@ -14,20 +14,74 @@ ==== Security Architecture +Unomi 3.1 authenticates every request so the server can resolve a **tenant** (or system-admin context) before profiles, events, and rules are touched. + +===== Authentication overview + +[cols="1,2,2",options="header"] +|=== +| Path class | Credentials | Typical use + +| Public (`/cxs/context.json`, `/cxs/eventcollector`, `/cxs/client/*`, …) +| `X-Unomi-Api-Key: ` +| Browser / mobile SDKs, trackers + +| Private REST +| Basic `tenantId:privateApiKey`, or JAAS + optional `X-Unomi-Tenant-Id` +| Server-side integrations, admin UIs + +| Tenant administration (`/cxs/tenants`) +| JAAS system administrator (for example `karaf:karaf`) +| Create tenants, rotate keys +|=== + +Temporary exception: <<_v2_compatibility_mode,V2 compatibility mode>> allows public endpoints without API keys while migrating 2.x clients. + +===== Auth resolution sequence + +[plantuml] +---- +@startuml +skinparam shadowing false +hide footbox + +actor Client +participant "AuthenticationFilter" as Auth +participant "TenantService" as Tenant +participant "REST endpoint" as API + +Client -> Auth : HTTP request +alt /cxs/tenants* + Auth -> Auth : Require JAAS admin + Auth -> API : System subject +else Public path + X-Unomi-Api-Key + Auth -> Tenant : Resolve tenant from public key + Auth -> API : Tenant context +else Private path + Basic tenantId:privateKey + Auth -> Tenant : Validate private key + Auth -> API : Tenant context +else Private path + JAAS (+ optional X-Unomi-Tenant-Id) + Auth -> API : Admin / tenant context +else V2 compatibility mode + Auth -> Tenant : Default tenant + Auth -> API : Legacy 2.x-style access +else + Auth --> Client : 401 Unauthorized +end +@enduml +---- + +===== Component view + [plantuml] ---- @startuml skinparam componentStyle uml2 -skinparam component { - BackgroundColor<> LightYellow - BackgroundColor<> LightBlue -} package "Security Layer" { [Security Service] <> [Authentication Filter] <> [Authorization Filter] <> - [Encryption Service] <> } package "API Layer" { @@ -42,7 +96,7 @@ package "Core Services" { } actor "Public Client" as public -actor "Admin Client" as admin +actor "Admin / Tenant Client" as admin database "Tenant Store" { [Tenant Configuration] @@ -59,43 +113,56 @@ admin --> [Private API] [Authorization Filter] --> [Security Service] [Security Service] --> [Tenant Configuration] [Security Service] --> [Roles & Permissions] - [Security Service] --> [Core Services] -[Encryption Service] --> [Profile Service] -note right of [Security Service] - - JAAS Authentication - - Role-based Authorization - - Tenant Isolation - - Operation Permissions +note right of [Authentication Filter] + - X-Unomi-Api-Key (public) + - Basic Auth (tenant or JAAS) + - X-Unomi-Tenant-Id (with JAAS) end note -note right of [Authentication Filter] - - Basic Auth - - API Key Auth - - Token Auth +note right of [Security Service] + - Role-based authorization + - Tenant isolation + - Operation permissions end note @enduml ---- -For tenant API keys, roles (`ROLE_UNOMI_TENANT_USER`, `ROLE_UNOMI_TENANT_ADMIN`, system administrator), and operation permissions, see <<_multitenancy,Multi-tenancy>>. - -===== Overview - -==== Authentication overview +Configuration keys live in `etc/org.apache.unomi.rest.authentication.cfg` and `custom.system.properties`. Tenant API key settings are in `etc/org.apache.unomi.tenant.cfg`. -Unomi 3.1 uses a layered authentication model: +For tenant API keys, roles (`ROLE_UNOMI_TENANT_USER`, `ROLE_UNOMI_TENANT_ADMIN`, system administrator), operation permissions, and curl examples, see <<_multitenancy,Multi-tenancy>> and <<_configuration,Configuration>> (tenant management). For migration from 2.x, see <<_v2_compatibility_mode,V2 compatibility mode>> and <<_migrate_from_3_0_to_3_1,Migrate from 3.0 to 3.1>>. -* *Public endpoints* (for example `/cxs/context.json`, `/cxs/eventcollector`): require a tenant **public** API key via the `X-Unomi-Api-Key` header. -* *Private REST endpoints*: require tenant `tenantId:privateApiKey` basic authentication, or JAAS credentials with the `X-Unomi-Tenant-Id` header. -* *System administrator*: Karaf JAAS user (for example `karaf:karaf`) for tenant management and cluster operations. +===== Multi-tenancy isolation -Configuration keys live in `etc/org.apache.unomi.rest.authentication.cfg` and `custom.system.properties`. Tenant API key settings are in `etc/org.apache.unomi.tenant.cfg`. +[plantuml] +---- +@startuml +skinparam shadowing false + +rectangle "Unomi instance" { + rectangle "Tenant A" as A { + card "Public key" as Ap + card "Private key" as Apr + database "Profiles / events / rules" as Ad + } + rectangle "Tenant B" as B { + card "Public key" as Bp + card "Private key" as Bpr + database "Profiles / events / rules" as Bd + } +} -For endpoint lists, auth examples, and migration from 2.x/3.0, see <<_multitenancy,Multi-tenancy>>, <<_configuration,Tenant management in Configuration>>, and <<_v2_compatibility_mode,V2 compatibility mode>>. +Ap -[hidden]-> Apr +Bp -[hidden]-> Bpr +A -[hidden]-> B +note bottom of A + No cross-tenant reads/writes +end note +@enduml +---- ==== Request tracing Administrators with the appropriate roles can enable request tracing with `explain=true` on context and event-collector requests. See <<_request_tracing_explain,Using the explain parameter for request tracing>>. - diff --git a/manual/src/main/asciidoc/shell-commands.adoc b/manual/src/main/asciidoc/shell-commands.adoc index 801a82de8b..19a09dca6f 100644 --- a/manual/src/main/asciidoc/shell-commands.adoc +++ b/manual/src/main/asciidoc/shell-commands.adoc @@ -42,7 +42,7 @@ karaf@root()> help unomi:migrate DESCRIPTION unomi:migrate - This will Migrate your date in ES to be compliant with current version. + This will migrate your data in Elasticsearch or OpenSearch to be compliant with the current version. It's possible to configure the migration using OSGI configuration file: org.apache.unomi.migration.cfg, if no configuration is provided then questions will be prompted during the migration process. diff --git a/manual/src/main/asciidoc/tutorial.adoc b/manual/src/main/asciidoc/tutorial.adoc index cc5d89abec..f381ce6d15 100644 --- a/manual/src/main/asciidoc/tutorial.adoc +++ b/manual/src/main/asciidoc/tutorial.adoc @@ -11,12 +11,21 @@ // See the License for the specific language governing permissions and // limitations under the License. // +[#_unomi_web_tracking_tutorial] === Unomi web tracking tutorial In this tutorial we will guide through the basic steps of getting started with a web tracking project. You will see how to integrate the built-in web tracker with an existing web site and what this enables. If you prefer to use existing HTML and Javascript rather than building your own, all the code we feature in this tutorial is extracted from our tracker sample which is available here: https://github.com/apache/unomi/blob/master/extensions/web-tracker/wab/src/main/webapp/index.html . However you will still need to use the REST API calls to create the scope and rule to make it all work. +IMPORTANT: On Unomi **3.1**, `/cxs/context.json` and `/cxs/eventcollector` require a tenant **public** API key. +The bundled `unomi-web-tracker` does not yet send `X-Unomi-Api-Key`. For this tutorial, either: + +* Enable <<_v2_compatibility_mode,V2 compatibility mode>> for local learning (public endpoints work without API keys), **or** +* Use the NPM tracker / custom fetch wrappers that set `X-Unomi-Api-Key` (see <<_javascript_tracker_guide,JavaScript tracker guide>>), after creating a tenant and regenerating keys (see <<_multitenancy,Multi-tenancy>>). + +Create a scope and rules with tenant private-key or JAAS admin authentication as shown below. + ==== Installing the web tracker in a web page Using the built-in tracker is pretty simple, simply add the following code to your HTML page : @@ -33,7 +42,7 @@ or you can also use the non-minified version that is available here: ---- -This will only load the tracker. To initialize it use a snipper like the following code: +This will only load the tracker. To initialize it use a snippet like the following code: [source,javascript] ---- @@ -59,7 +68,7 @@ This will only load the tracker. To initialize it use a snipper like the followi "attributes": {}, "consentTypes": [] }, - "events:": [], + "events": [], "wemInitConfig": { "contextServerUrl": document.location.origin, "timeoutInMilliseconds": "1500", @@ -149,7 +158,7 @@ import {useTracker} from "apache-unomi-tracker"; "attributes": {}, "consentTypes": [] }, - "events:": [], + "events": [], "wemInitConfig": { "contextServerUrl": document.location.origin, "timeoutInMilliseconds": "1500", diff --git a/manual/src/main/asciidoc/upgrades/upgrade-dependency-hygiene.adoc b/manual/src/main/asciidoc/upgrades/upgrade-dependency-hygiene.adoc index 99dd6fc18e..b3090aa85d 100644 --- a/manual/src/main/asciidoc/upgrades/upgrade-dependency-hygiene.adoc +++ b/manual/src/main/asciidoc/upgrades/upgrade-dependency-hygiene.adoc @@ -45,13 +45,11 @@ The bulk dependency pass https://issues.apache.org/jira/browse/UNOMI-829[UNOMI-8 ===== Recommended order (platform stack) - ===== Hygiene vs platform upgrade scope [plantuml] ---- @startuml -skinparam activityBackgroundColor #F8F8F8 start :Library bump requested; diff --git a/manual/src/main/asciidoc/upgrades/upgrade-elasticsearch.adoc b/manual/src/main/asciidoc/upgrades/upgrade-elasticsearch.adoc index f1d1d15466..f83cb745fc 100644 --- a/manual/src/main/asciidoc/upgrades/upgrade-elasticsearch.adoc +++ b/manual/src/main/asciidoc/upgrades/upgrade-elasticsearch.adoc @@ -56,11 +56,6 @@ Image tag example: `docker.elastic.co/elasticsearch/elasticsearch:9.4.3`. ---- @startuml skinparam componentStyle uml2 -skinparam component { - BackgroundColor<> #E8F4E8 - BackgroundColor<> #E8EEF8 - BackgroundColor<> #F0F0F0 -} package "Karaf platform" <> { [slf4j-api provided] diff --git a/manual/src/main/asciidoc/upgrades/upgrade-integration-tests.adoc b/manual/src/main/asciidoc/upgrades/upgrade-integration-tests.adoc index 1b40307e6e..107b375d2c 100644 --- a/manual/src/main/asciidoc/upgrades/upgrade-integration-tests.adoc +++ b/manual/src/main/asciidoc/upgrades/upgrade-integration-tests.adoc @@ -20,13 +20,11 @@ Platform upgrades are validated by *Pax Exam* integration tests plus *Docker-bac See also: `itests/README.md` and <<_using_the_build_script,build.sh>>. - ===== Validation pipeline after a platform bump [plantuml] ---- @startuml -skinparam activityBackgroundColor #F8F8F8 |Maintainer| start diff --git a/manual/src/main/asciidoc/upgrades/upgrade-java.adoc b/manual/src/main/asciidoc/upgrades/upgrade-java.adoc index 790f8105e5..30f1668f2b 100644 --- a/manual/src/main/asciidoc/upgrades/upgrade-java.adoc +++ b/manual/src/main/asciidoc/upgrades/upgrade-java.adoc @@ -30,7 +30,6 @@ A JDK upgrade is almost always bundled with a *major* Karaf migration. Unomi 3.x Also update CI workflows, `build.sh` preflight checks, and contributor docs when raising the minimum JDK. - ===== JDK upgrade coupled to Karaf (diagram) Java version changes are not done in isolation on Unomi 3.x — they ride the major Karaf migration path (UNOMI-876). @@ -38,7 +37,6 @@ Java version changes are not done in isolation on Unomi 3.x — they ride the ma [plantuml] ---- @startuml -skinparam activityBackgroundColor #F8F8F8 start :Read Karaf release notes diff --git a/manual/src/main/asciidoc/upgrades/upgrade-jira-reference.adoc b/manual/src/main/asciidoc/upgrades/upgrade-jira-reference.adoc index 628c624c7e..c0df8bd9a4 100644 --- a/manual/src/main/asciidoc/upgrades/upgrade-jira-reference.adoc +++ b/manual/src/main/asciidoc/upgrades/upgrade-jira-reference.adoc @@ -124,7 +124,6 @@ IMPORTANT: https://issues.apache.org/jira/browse/UNOMI-884[UNOMI-884] covers *in ===== Historical blockers (Karaf) - ===== Karaf upgrade prerequisites (timeline) Historical JIRA blockers explain why Karaf 4.3+ waited until Unomi 3 clustering changed: @@ -132,7 +131,6 @@ Historical JIRA blockers explain why Karaf 4.3+ waited until Unomi 3 clustering [plantuml] ---- @startuml -skinparam activityBackgroundColor #F8F8F8 title Karaf upgrade path (JIRA milestones) diff --git a/manual/src/main/asciidoc/upgrades/upgrade-karaf.adoc b/manual/src/main/asciidoc/upgrades/upgrade-karaf.adoc index c45e5333e9..dba1ccdd3b 100644 --- a/manual/src/main/asciidoc/upgrades/upgrade-karaf.adoc +++ b/manual/src/main/asciidoc/upgrades/upgrade-karaf.adoc @@ -42,7 +42,6 @@ TIP: JIRA context for prerequisites (Cellar removal, CXF 3.x ceiling, UNOMI-876 [plantuml] ---- @startuml -skinparam activityBackgroundColor #F8F8F8 start :New target Karaf version; diff --git a/manual/src/main/asciidoc/upgrades/upgrade-overview.adoc b/manual/src/main/asciidoc/upgrades/upgrade-overview.adoc index 21c3315f1f..f4658c0a7c 100644 --- a/manual/src/main/asciidoc/upgrades/upgrade-overview.adoc +++ b/manual/src/main/asciidoc/upgrades/upgrade-overview.adoc @@ -70,8 +70,6 @@ Recommended order when bumping multiple platform components in one release cycle [plantuml] ---- @startuml -skinparam activityBackgroundColor #F8F8F8 -skinparam activityBorderColor #333333 start :Identify bump scope @@ -130,12 +128,6 @@ Karaf is the hub: most runtime libraries come from Karaf features, while Unomi M ---- @startuml skinparam componentStyle uml2 -skinparam component { - BackgroundColor<> #E8F4E8 - BackgroundColor<> #FFF4E6 - BackgroundColor<> #E8EEF8 - BackgroundColor<> #F0F0F0 -} package "Apache Karaf (karaf.version)" <> { [Pax Web / Jetty runtime] diff --git a/manual/src/main/asciidoc/useful-unomi-urls.adoc b/manual/src/main/asciidoc/useful-unomi-urls.adoc index e1bc007330..804c96e878 100644 --- a/manual/src/main/asciidoc/useful-unomi-urls.adoc +++ b/manual/src/main/asciidoc/useful-unomi-urls.adoc @@ -112,7 +112,7 @@ where PROFILE_ID is a profile identifier. This will indeed retrieve all the even |/cxs/tenants |POST -|Create a tenant (system administrator). Returns generated public and private API keys. +|Create a tenant (system administrator). Returns the tenant with **masked** API key metadata; obtain plaintext via `POST /cxs/tenants/{id}/apikeys`. |/cxs/tenants/{tenantId}/apikeys |POST diff --git a/manual/src/main/asciidoc/web-tracker.adoc b/manual/src/main/asciidoc/web-tracker.adoc index 0ec98d0fe1..c2808446c3 100644 --- a/manual/src/main/asciidoc/web-tracker.adoc +++ b/manual/src/main/asciidoc/web-tracker.adoc @@ -16,6 +16,17 @@ In this section of the documentation, more details are provided about the web tracker provided by Unomi. +==== Authentication (Unomi 3.1) + +Public context and event endpoints require a tenant public API key (`X-Unomi-Api-Key`) unless <<_v2_compatibility_mode,V2 compatibility mode>> is enabled. + +The built-in WAB tracker sample does **not** currently attach that header. For production 3.1 deployments: + +* Prefer the NPM package / custom integration that can set `X-Unomi-Api-Key` (see <<_javascript_tracker_guide,JavaScript tracker guide>>), or +* Enable V2 compatibility mode only as a temporary migration aid. + +Create a tenant and obtain plaintext public/private keys via `POST /cxs/tenants/{id}/apikeys` before integrating a site. See <<_multitenancy,Multi-tenancy>> and the <<_unomi_web_tracking_tutorial,web tracking tutorial>>. + ==== Custom events In order to be able to use your own custom events with the web tracker, you must first declare them in Unomi so that they are properly recognized and validated by the `/context.json` or `/eventcollector` endpoints. diff --git a/manual/src/main/asciidoc/whats-new.adoc b/manual/src/main/asciidoc/whats-new.adoc index 812dcff4bd..4b34f3f906 100644 --- a/manual/src/main/asciidoc/whats-new.adoc +++ b/manual/src/main/asciidoc/whats-new.adoc @@ -38,10 +38,6 @@ Official alternative to Elasticsearch, including secured Docker deployments. * Configuration: <<_configuration,Configuration>> * Migration from Elasticsearch: <<_migrate_from_elasticsearch_to_opensearch,Migrate from Elasticsearch to OpenSearch>> -==== Persistence-based clustering - -Cluster node registry and heartbeats through Elasticsearch or OpenSearch (no Karaf Cellar). See <<_clustering,Cluster setup>>. - ==== Tenant usage and retention Read-only per-tenant usage metrics and an event retention purge API for upstream control planes. diff --git a/manual/src/main/asciidoc/writing-plugins.adoc b/manual/src/main/asciidoc/writing-plugins.adoc index c49808422e..4574ccf960 100644 --- a/manual/src/main/asciidoc/writing-plugins.adoc +++ b/manual/src/main/asciidoc/writing-plugins.adoc @@ -626,7 +626,7 @@ services: elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:9.4.3 opensearch: - image: opensearchproject/opensearch:3.4.0 + image: opensearchproject/opensearch:3.7.0 ---- ==== Reference implementations in the core codebase diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java index 1b5a683e95..069aac41ff 100644 --- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java +++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java @@ -24,7 +24,7 @@ import org.apache.karaf.shell.api.console.Session; import org.apache.unomi.shell.migration.MigrationService; -@Command(scope = "unomi", name = "migrate", description = "This will Migrate your data in ES to be compliant with current version. " + +@Command(scope = "unomi", name = "migrate", description = "This will migrate your data in Elasticsearch or OpenSearch to be compliant with the current version. " + "It's possible to configure the migration using OSGI configuration file: org.apache.unomi.migration.cfg, " + "if no configuration is provided then questions will be prompted during the migration process.") @Service