From 4bb7edc747d78192e897de3185abf4cab3981e1f Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 3 Nov 2026 16:22:08 +0100 Subject: [PATCH 1/5] add explanation of token usage for logins, API, DAGs etc. --- .../usage-guide/dag-run-information.adoc | 49 +++++++++++++++++ .../airflow/pages/usage-guide/security.adoc | 54 +++++++++++++++++++ docs/modules/airflow/partials/nav.adoc | 1 + 3 files changed, 104 insertions(+) create mode 100644 docs/modules/airflow/pages/usage-guide/dag-run-information.adoc diff --git a/docs/modules/airflow/pages/usage-guide/dag-run-information.adoc b/docs/modules/airflow/pages/usage-guide/dag-run-information.adoc new file mode 100644 index 00000000..f54d62aa --- /dev/null +++ b/docs/modules/airflow/pages/usage-guide/dag-run-information.adoc @@ -0,0 +1,49 @@ += Reading DAG run information in Airflow 3 +:description: Replace direct metadata database access in DAGs with the Airflow REST API. +:airflow-upgrade-docs: https://airflow.apache.org/docs/apache-airflow/stable/installation/upgrading_to_airflow3.html + +In Airflow 3, task code can no longer access the Airflow metadata database directly, so queries such as `session.query(DagRun)` no longer work. +See the {airflow-upgrade-docs}[Airflow 3 upgrade guide{external-link-icon}^] for background. + +DAGs that monitor other DAGs can use the REST API instead, authenticating as a technical user as described in xref:usage-guide/security.adoc#api-access[REST API access]: + +[source,python] +---- +import os + +import pendulum +import requests +from airflow.sdk import dag, task + +AIRFLOW_URL = "http://-webserver:8080" + + +@dag(schedule=None, start_date=pendulum.datetime(2026, 1, 1, tz="UTC"), catchup=False) +def monitor_dag_runs(): + @task + def report(dag_id: str): + response = requests.post( + f"{AIRFLOW_URL}/auth/token", + json={ + "username": os.environ["AIRFLOW_API_USERNAME"], + "password": os.environ["AIRFLOW_API_PASSWORD"], + }, + timeout=30, + ) + response.raise_for_status() + headers = {"Authorization": f"Bearer {response.json()['access_token']}"} + + response = requests.get( + f"{AIRFLOW_URL}/api/v2/dags/{dag_id}/dagRuns", headers=headers, timeout=30 + ) + response.raise_for_status() + for run in response.json()["dag_runs"]: + print(run["dag_run_id"], run["state"], run["start_date"], run["end_date"]) + + report("") + + +monitor_dag_runs() +---- + +The environment variables can be populated from a Secret holding the technical user's credentials with xref:usage-guide/overrides.adoc#_pod_overrides[Pod overrides]. diff --git a/docs/modules/airflow/pages/usage-guide/security.adoc b/docs/modules/airflow/pages/usage-guide/security.adoc index 3e4dadd8..33a0d8ac 100644 --- a/docs/modules/airflow/pages/usage-guide/security.adoc +++ b/docs/modules/airflow/pages/usage-guide/security.adoc @@ -2,6 +2,7 @@ :description: Secure Apache Airflow by configuring user authentication and authorization. :airflow-access-control-docs: https://airflow.apache.org/docs/apache-airflow-providers-fab/stable/auth-manager/access-control.html :keycloak: https://www.keycloak.org/ +:airflow-fab-token-docs: https://airflow.apache.org/docs/apache-airflow-providers-fab/stable/auth-manager/token.html Secure Apache Airflow by configuring user authentication and authorization. Airflow provides built-in user and role management, but can also connect to an LDAP server or an OIDC provider to manage users centrally instead. @@ -140,6 +141,59 @@ A minimum client configuration in Keycloak for this example looks like this: Further information for specifying an AuthenticationClass for an OIDC provider can be found at the xref:concepts:authentication.adoc#_oidc[concepts page]. +=== [[api-access]]REST API access + +This section applies to Airflow 3.x. + +The authentication methods described above (built-in users, LDAP and OpenID Connect) govern how users log in to the web interface. +The REST API (`/api/v2`) authenticates requests differently: it accepts only JSON Web Tokens (JWTs) issued by Airflow itself. +A client obtains such a token by sending a username and password to the `/auth/token` endpoint of the webserver: + +[source,bash] +---- +ACCESS_TOKEN=$( + curl -s -X POST http://-webserver:8080/auth/token \ + -H 'Content-Type: application/json' \ + -d '{"username": "", "password": ""}' | jq -r '.access_token' +) +curl -s -H "Authorization: Bearer $ACCESS_TOKEN" \ + http://-webserver:8080/api/v2/dags//dagRuns +---- + +The `/auth/token` endpoint checks the credentials against Airflow's internal user database (and against the LDAP server if LDAP authentication is configured). +See the {airflow-fab-token-docs}[Airflow documentation on generating tokens{external-link-icon}^] for details. + +IMPORTANT: With OpenID Connect, Keycloak is used for the interactive login to the web interface only. +Tokens issued by Keycloak (for example via the client credentials grant of a service account) are rejected by the REST API with `Invalid JWT token`. +Users created through a Keycloak login have no usable password in Airflow, so their credentials are rejected by `/auth/token` with `Invalid credentials`. + +NOTE: Airflow allows tokens to be obtained for OAuth users by overriding the `create_token` method of the FAB auth manager, as described in the {airflow-fab-token-docs}[Airflow documentation on generating tokens{external-link-icon}^]. +The Stackable Data Platform does not provide or test such an override. +It requires a custom auth manager in a custom image, and that code is then responsible for validating the token from the identity provider (signature, issuer, audience and expiry). +The example in the Airflow documentation disables signature verification and must not be used in production. + +==== Service-to-service access + +For API clients such as scripts, external services or DAGs, create a dedicated technical user in Airflow's internal user database. +This works regardless of the authentication method configured for the web interface. +Assign the least privileged role that covers the required API calls; for example, the `Viewer` role allows reading DAGs and DAG runs, but not modifying them: + +[source,bash] +---- +kubectl exec -webserver--0 -c airflow -- \ + airflow users create \ + --username \ + --password \ + --role Viewer \ + --firstname \ + --lastname \ + --email +---- + +Store the credentials in a Kubernetes Secret and make them available to the client. +Request a new token for each run rather than storing one, as tokens expire. +See xref:usage-guide/dag-run-information.adoc[] for an example of a DAG that reads DAG run information via the REST API. + == Authorization The Airflow Webserver delegates the {airflow-access-control-docs}[handling of user access control{external-link-icon}^] to the https://flask-appbuilder.readthedocs.io/en/latest/security.html[Flask AppBuilder{external-link-icon}^]. diff --git a/docs/modules/airflow/partials/nav.adoc b/docs/modules/airflow/partials/nav.adoc index 30993bf1..40b67b10 100644 --- a/docs/modules/airflow/partials/nav.adoc +++ b/docs/modules/airflow/partials/nav.adoc @@ -11,6 +11,7 @@ ** xref:airflow:usage-guide/reverse-proxy.adoc[] ** xref:airflow:usage-guide/storage-resources.adoc[] ** xref:airflow:usage-guide/security.adoc[] +** xref:airflow:usage-guide/dag-run-information.adoc[] ** xref:airflow:usage-guide/logging.adoc[] ** xref:airflow:usage-guide/monitoring.adoc[] ** xref:airflow:usage-guide/openlineage.adoc[] From 9eb405e6d243c20060bd016b51c6435bdac1295f Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 3 Nov 2026 16:29:39 +0100 Subject: [PATCH 2/5] remove clause --- docs/modules/airflow/pages/usage-guide/security.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/airflow/pages/usage-guide/security.adoc b/docs/modules/airflow/pages/usage-guide/security.adoc index 33a0d8ac..2d7d1789 100644 --- a/docs/modules/airflow/pages/usage-guide/security.adoc +++ b/docs/modules/airflow/pages/usage-guide/security.adoc @@ -169,7 +169,7 @@ Users created through a Keycloak login have no usable password in Airflow, so th NOTE: Airflow allows tokens to be obtained for OAuth users by overriding the `create_token` method of the FAB auth manager, as described in the {airflow-fab-token-docs}[Airflow documentation on generating tokens{external-link-icon}^]. The Stackable Data Platform does not provide or test such an override. -It requires a custom auth manager in a custom image, and that code is then responsible for validating the token from the identity provider (signature, issuer, audience and expiry). +It requires a custom auth manager in a custom image, and that code is then responsible for validating the token from the identity provider. The example in the Airflow documentation disables signature verification and must not be used in production. ==== Service-to-service access From 1da03b0e5fcd2c94552989078bb18a9ae577567b Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Thu, 5 Nov 2026 10:41:52 +0100 Subject: [PATCH 3/5] Update docs/modules/airflow/pages/usage-guide/dag-run-information.adoc Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com> --- docs/modules/airflow/pages/usage-guide/dag-run-information.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/airflow/pages/usage-guide/dag-run-information.adoc b/docs/modules/airflow/pages/usage-guide/dag-run-information.adoc index f54d62aa..dd2e243f 100644 --- a/docs/modules/airflow/pages/usage-guide/dag-run-information.adoc +++ b/docs/modules/airflow/pages/usage-guide/dag-run-information.adoc @@ -46,4 +46,4 @@ def monitor_dag_runs(): monitor_dag_runs() ---- -The environment variables can be populated from a Secret holding the technical user's credentials with xref:usage-guide/overrides.adoc#_pod_overrides[Pod overrides]. +The environment variables can be populated from a Secret holding the user's credentials with xref:usage-guide/overrides.adoc#_pod_overrides[Pod overrides]. From d10b7bd7c76fa3ceb7875db0ac795bce2aef4c43 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Thu, 5 Nov 2026 10:42:14 +0100 Subject: [PATCH 4/5] Update docs/modules/airflow/pages/usage-guide/security.adoc Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com> --- docs/modules/airflow/pages/usage-guide/security.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/airflow/pages/usage-guide/security.adoc b/docs/modules/airflow/pages/usage-guide/security.adoc index 2d7d1789..99a84bdd 100644 --- a/docs/modules/airflow/pages/usage-guide/security.adoc +++ b/docs/modules/airflow/pages/usage-guide/security.adoc @@ -174,7 +174,7 @@ The example in the Airflow documentation disables signature verification and mus ==== Service-to-service access -For API clients such as scripts, external services or DAGs, create a dedicated technical user in Airflow's internal user database. +For API clients such as scripts, external services or DAGs, create a dedicated user in Airflow's internal user database. This works regardless of the authentication method configured for the web interface. Assign the least privileged role that covers the required API calls; for example, the `Viewer` role allows reading DAGs and DAG runs, but not modifying them: From 88f7a17e25d9bad1d9179650a2bad08aa3e2aa77 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 5 Nov 2026 10:48:12 +0100 Subject: [PATCH 5/5] add admonition --- docs/modules/airflow/pages/usage-guide/security.adoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/modules/airflow/pages/usage-guide/security.adoc b/docs/modules/airflow/pages/usage-guide/security.adoc index 99a84bdd..2cd3e39c 100644 --- a/docs/modules/airflow/pages/usage-guide/security.adoc +++ b/docs/modules/airflow/pages/usage-guide/security.adoc @@ -170,7 +170,8 @@ Users created through a Keycloak login have no usable password in Airflow, so th NOTE: Airflow allows tokens to be obtained for OAuth users by overriding the `create_token` method of the FAB auth manager, as described in the {airflow-fab-token-docs}[Airflow documentation on generating tokens{external-link-icon}^]. The Stackable Data Platform does not provide or test such an override. It requires a custom auth manager in a custom image, and that code is then responsible for validating the token from the identity provider. -The example in the Airflow documentation disables signature verification and must not be used in production. + +IMPORTANT: The example in the Airflow documentation disables signature verification and (as also stated there) should not be used in production. ==== Service-to-service access