From a6f92cb6c20ab85204e4a0c614700ce951d66938 Mon Sep 17 00:00:00 2001 From: Ryan Eno Date: Wed, 26 Aug 2026 20:18:05 -0700 Subject: [PATCH 1/2] feat: add environment-backed output target --- .agents/skills/espipe/SKILL.md | 4 +- CHANGELOG.md | 8 ++ Cargo.lock | 7 ++ Cargo.toml | 1 + README.md | 12 +- .../.openspec.yaml | 2 + .../design.md | 51 +++++++++ .../proposal.md | 28 +++++ .../elasticsearch-environment-output/spec.md | 99 +++++++++++++++++ .../tasks.md | 11 ++ .../elasticsearch-environment-output/spec.md | 99 +++++++++++++++++ src/main.rs | 32 ++++-- src/output/mod.rs | 71 +++++------- tests/env_output.rs | 105 ++++++++++++++++++ tests/index_template.rs | 16 ++- 15 files changed, 485 insertions(+), 61 deletions(-) create mode 100644 openspec/changes/archive/2026-08-26-add-elasticsearch-environment-output/.openspec.yaml create mode 100644 openspec/changes/archive/2026-08-26-add-elasticsearch-environment-output/design.md create mode 100644 openspec/changes/archive/2026-08-26-add-elasticsearch-environment-output/proposal.md create mode 100644 openspec/changes/archive/2026-08-26-add-elasticsearch-environment-output/specs/elasticsearch-environment-output/spec.md create mode 100644 openspec/changes/archive/2026-08-26-add-elasticsearch-environment-output/tasks.md create mode 100644 openspec/specs/elasticsearch-environment-output/spec.md create mode 100644 tests/env_output.rs diff --git a/.agents/skills/espipe/SKILL.md b/.agents/skills/espipe/SKILL.md index 39388aa..d49a2db 100644 --- a/.agents/skills/espipe/SKILL.md +++ b/.agents/skills/espipe/SKILL.md @@ -38,7 +38,7 @@ Elasticsearch outputs: - `http://host:9200/index-name` - `https://host:9200/index-name` - `known-host:index-name`, resolved from `$ESPIPE_HOSTS` or `~/.espipe/hosts.yml` -- `elasticsearch:/index-name` or `es:/index-name`, resolved with `ELASTIC_ES_URL` and optionally `ELASTIC_ES_API_KEY` +- `env:/index-name`, resolved first from the process environment and then from `.env` using `ELASTIC_ES_URL` and optionally `ELASTIC_ES_API_KEY` Other outputs: @@ -89,7 +89,7 @@ Examples: - `espipe accounts.csv records:customers` - `espipe users.csv https://host:9200/users` -- `espipe --action upsert --generate-id=true 'docs/**/*.md' elasticsearch:/documents` +- `espipe --action upsert --generate-id=true 'docs/**/*.md' env:/documents` - `espipe --split /hits response.json output.ndjson` Use only flags the user requests or that are required to express the destination. Do not reinterpret `--action index` as an overwrite-by-source-ID option; IDs are used only when explicit or generated according to the rules above. diff --git a/CHANGELOG.md b/CHANGELOG.md index 116e0a7..d9336c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Added `.env` fallback for missing `ELASTIC_ES_URL` and `ELASTIC_ES_API_KEY` environment settings. + +### Changed + +- Replaced the `elasticsearch:/index` and `es:/index` environment targets with the explicit `env:/index` form. + ## [0.6.1] - 2026-08-22 ### Changed diff --git a/Cargo.lock b/Cargo.lock index d5b3506..8bb76ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -685,6 +685,12 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dunce" version = "1.0.5" @@ -792,6 +798,7 @@ dependencies = [ "bytes", "clap", "csv", + "dotenvy", "elasticsearch", "env_logger", "eyre", diff --git a/Cargo.toml b/Cargo.toml index f4f9649..e5c70df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ anydoc = "^0.2.3" base64 = "^0.23.1" clap = { version = "^4.6.6", features = ["derive"] } csv = "^1.4.0" +dotenvy = "^0.15.7" elasticsearch = "^9.1.0-alpha.1" env_logger = "^0.11.11" eyre = "^0.6.14" diff --git a/README.md b/README.md index daea2d9..44047a2 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,8 @@ Multi-file and glob imports log per-file read or conversion failures and continu Sends documents to Elasticsearch over TLS. - `known-host:index-name` Resolves `known-host` from a local hosts file and sends to the named index. +- `env:/index-name` + Reads the cluster URL and optional API key from environment variables or `.env`. When writing to Elasticsearch, the output path must include an index name. @@ -242,17 +244,17 @@ espipe docs.ndjson ess-cluster:my-index Known-host outputs use the authentication and TLS settings from their host entry. -### Elastic CLI contexts +### Environment targets -As an [Elastic CLI extension](https://github.com/elastic/cli), `espipe` reads the active Elasticsearch context from: +The `env:/index` output reads its connection settings from: - `ELASTIC_ES_URL` supplies the Elasticsearch base URL. - `ELASTIC_ES_API_KEY` supplies API-key authentication when no `--apikey`, `--username`, or `--password` option is provided. -Use `elasticsearch:/index` or `es:/index` as the output. These schemes take precedence over same-named known hosts. +Values already present in the process environment take precedence. For missing values, `espipe` searches the current directory and its parents for a `.env` file. The command fails if `ELASTIC_ES_URL` remains unset. This also works with environment variables supplied by an [Elastic CLI extension](https://github.com/elastic/cli). ```bash -espipe docs.ndjson es:/my-index +espipe docs.ndjson env:/my-index ``` ## Examples @@ -303,7 +305,7 @@ espipe docs.ndjson https://example.com:9200/my-index \ ### Use the active Elastic CLI context ```bash -elastic espipe docs.ndjson es:/my-index +elastic espipe docs.ndjson env:/my-index ``` ### Tune bulk requests diff --git a/openspec/changes/archive/2026-08-26-add-elasticsearch-environment-output/.openspec.yaml b/openspec/changes/archive/2026-08-26-add-elasticsearch-environment-output/.openspec.yaml new file mode 100644 index 0000000..701445b --- /dev/null +++ b/openspec/changes/archive/2026-08-26-add-elasticsearch-environment-output/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-26 diff --git a/openspec/changes/archive/2026-08-26-add-elasticsearch-environment-output/design.md b/openspec/changes/archive/2026-08-26-add-elasticsearch-environment-output/design.md new file mode 100644 index 0000000..71f2a13 --- /dev/null +++ b/openspec/changes/archive/2026-08-26-add-elasticsearch-environment-output/design.md @@ -0,0 +1,51 @@ +## Context + +Output dispatch treats HTTP and HTTPS schemes as direct Elasticsearch targets, `file` as local output, and other schemes as configured host names. The previous environment-backed branch reserved both `es` and `elasticsearch` before configured-host lookup. Environment settings came only from the process environment. See `proposal.md` for the reason for changing that behavior. + +The new capability crosses command startup and output dispatch, and it adds one dependency for `.env` parsing. + +## Goals / Non-Goals + +**Goals:** + +- Make the configuration source visible in the output URI. +- Keep process environment values authoritative while filling missing values from `.env`. +- Limit `.env` loading and Elastic environment authentication to `env:/` output. +- Preserve the existing direct URL and configured-host paths. + +**Non-Goals:** + +- Add a flag for selecting a `.env` path. +- Add environment-backed input URIs. +- Change the names or formats of `ELASTIC_ES_URL` and `ELASTIC_ES_API_KEY`. +- Change known-host file discovery or authentication. + +## Decisions + +### Reserve only `env` + +Output dispatch checks for the exact `env` scheme before direct URL and configured-host handling. `es` and `elasticsearch` take the configured-host path. This makes the special behavior explicit and avoids permanently consuming plausible cluster aliases. + +Keeping the two existing schemes as deprecated aliases was considered. It would preserve compatibility, but it would keep the ambiguity and prevent those configured-host names from working. + +### Load `.env` only for environment output + +After parsing the output URI, command startup invokes `dotenvy` only when the scheme is `env`. The standard loader searches the working directory and its ancestors and does not replace variables already present in the process environment. Missing `.env` files are allowed. Parse and read errors other than absence stop startup. + +Loading `.env` unconditionally at process startup was considered. It could change logging or authentication for direct URL and configured-host commands, which is outside this capability. + +### Keep authentication precedence in command startup + +Command-line authentication remains authoritative. The resolved environment API key is passed to authentication setup only for `env:/` output and only when no command-line authentication option is present. This keeps output construction independent of argument precedence rules. + +### Validate and join the URL at the output boundary + +The environment output branch parses `ELASTIC_ES_URL`, requires an absolute HTTP or HTTPS URL with a host, appends the target index to the configured base path, and removes query and fragment components. It then reuses the direct Elasticsearch output builder. + +Building the target with string concatenation was considered, but URL parsing gives consistent validation and path handling before network work starts. + +## Risks / Trade-offs + +- [Existing commands using `es:/` or `elasticsearch:/` break] -> Document `env:/` as the migration and record the change as breaking. +- [A `.env` file in a parent directory supplies settings unexpectedly] -> Follow `dotenvy`'s documented nearest-file search and state that behavior in the capability spec and README. +- [Malformed `.env` content blocks an environment-backed command] -> Report the parsing error before input ingestion or network activity. diff --git a/openspec/changes/archive/2026-08-26-add-elasticsearch-environment-output/proposal.md b/openspec/changes/archive/2026-08-26-add-elasticsearch-environment-output/proposal.md new file mode 100644 index 0000000..77b54e6 --- /dev/null +++ b/openspec/changes/archive/2026-08-26-add-elasticsearch-environment-output/proposal.md @@ -0,0 +1,28 @@ +## Why + +The `es:/` and `elasticsearch:/` output schemes do not say that espipe resolves their connection settings from environment variables. They also prevent users from assigning those names to configured hosts. An explicit `env:/` target makes the configuration source clear and leaves ordinary host aliases available. + +## What changes + +- Add `env:/` as the Elasticsearch output form backed by `ELASTIC_ES_URL` and the optional `ELASTIC_ES_API_KEY`. +- Load missing environment settings from the nearest `.env` file without replacing values already present in the process environment. +- Fail with a clear error when `ELASTIC_ES_URL` remains unset or is not an absolute HTTP or HTTPS URL. +- Preserve explicit command-line authentication precedence over `ELASTIC_ES_API_KEY`. +- **BREAKING**: Stop reserving `es:/` and `elasticsearch:/`; resolve them as configured host names instead. + +## Capabilities + +### New capabilities + +- `elasticsearch-environment-output`: Defines the environment-backed output URI, setting precedence, URL validation, authentication precedence, and configured-host namespace behavior. + +### Modified capabilities + +None. + +## Impact + +- Affects output URI dispatch and environment authentication handling in `src/main.rs` and `src/output/mod.rs`. +- Adds the `dotenvy` runtime dependency. +- Adds CLI coverage for process environment, `.env`, and missing-setting behavior. +- Changes commands that used `es:/` or `elasticsearch:/` to use `env:/`. diff --git a/openspec/changes/archive/2026-08-26-add-elasticsearch-environment-output/specs/elasticsearch-environment-output/spec.md b/openspec/changes/archive/2026-08-26-add-elasticsearch-environment-output/specs/elasticsearch-environment-output/spec.md new file mode 100644 index 0000000..4af9fe0 --- /dev/null +++ b/openspec/changes/archive/2026-08-26-add-elasticsearch-environment-output/specs/elasticsearch-environment-output/spec.md @@ -0,0 +1,99 @@ +## Purpose + +Define how an `env:/` Elasticsearch output resolves connection settings while preserving configured host names and explicit command-line authentication. + +## ADDED Requirements + +### Requirement: Environment output uses an explicit URI form + +The system SHALL reserve `env:/` for an Elasticsearch output whose connection settings come from the process environment or `.env`. The URI SHALL contain one slash after `env:` and a non-empty index path. The system SHALL NOT reserve `es` or `elasticsearch` for environment-backed output. + +#### Scenario: Valid environment output is provided + +- **WHEN** the user provides `env:/logs` as the output +- **THEN** the system selects environment-backed Elasticsearch output +- **AND** it uses `logs` as the target index + +#### Scenario: Environment output omits the required index + +- **WHEN** the user provides `env:/` as the output +- **THEN** startup fails with an error that identifies `env:/index` as the required form + +#### Scenario: Environment output uses an authority or omits the slash + +- **WHEN** the user provides `env://logs` or `env:logs` as the output +- **THEN** startup fails with an error that identifies `env:/index` as the required form + +#### Scenario: Former environment scheme is used + +- **WHEN** the user provides an output whose scheme is `es` or `elasticsearch` +- **THEN** the system resolves that scheme as a configured host name +- **AND** it does not read Elastic environment settings for that output + +### Requirement: Environment settings use deterministic precedence + +For an `env:/` output, the system SHALL use values already present in the process environment. For each missing setting, it SHALL search the working directory and its ancestors for the nearest `.env` file and load the setting from that file. A `.env` value SHALL NOT replace a value already present in the process environment. + +#### Scenario: Process environment contains the URL + +- **WHEN** `ELASTIC_ES_URL` is present in the process environment +- **AND** `.env` contains a different `ELASTIC_ES_URL` +- **THEN** the system uses the value from the process environment + +#### Scenario: Dotenv supplies a missing URL + +- **WHEN** `ELASTIC_ES_URL` is absent from the process environment +- **AND** the nearest `.env` file defines `ELASTIC_ES_URL` +- **THEN** the system uses the value from `.env` + +#### Scenario: Dotenv file is malformed + +- **WHEN** the nearest `.env` file cannot be parsed +- **THEN** startup fails with an error that identifies `.env` as unreadable + +#### Scenario: Non-environment output is selected + +- **WHEN** the output does not use the `env` scheme +- **THEN** the system does not load `.env` for Elasticsearch connection settings + +### Requirement: Environment URL is required and valid + +An `env:/` output SHALL require `ELASTIC_ES_URL` after environment and `.env` resolution. The value SHALL be an absolute `http://` or `https://` URL with a host. The system SHALL append the output index to any existing base path and SHALL discard query and fragment components from the configured URL. + +#### Scenario: URL remains unset + +- **WHEN** neither the process environment nor `.env` defines `ELASTIC_ES_URL` +- **THEN** startup fails with an error that names `ELASTIC_ES_URL` + +#### Scenario: URL uses an unsupported or relative form + +- **WHEN** the resolved `ELASTIC_ES_URL` is relative, lacks a host, or uses a scheme other than HTTP or HTTPS +- **THEN** startup fails before sending documents + +#### Scenario: URL contains a base path + +- **WHEN** `ELASTIC_ES_URL` is `https://example.com/elasticsearch/?ignored=true#fragment` +- **AND** the output is `env:/logs` +- **THEN** the Elasticsearch output URL is `https://example.com/elasticsearch/logs` + +### Requirement: Explicit authentication takes precedence + +For an `env:/` output, the system SHALL use `ELASTIC_ES_API_KEY` from the process environment or `.env` when the user supplies no authentication option. An explicit `--apikey` or complete `--username` and `--password` pair SHALL take precedence over `ELASTIC_ES_API_KEY`. + +#### Scenario: Environment API key is the only authentication setting + +- **WHEN** `ELASTIC_ES_API_KEY` resolves from the process environment or `.env` +- **AND** the user supplies no authentication option +- **THEN** the system authenticates with the resolved API key + +#### Scenario: Explicit API key is provided + +- **WHEN** the user supplies `--apikey` +- **AND** `ELASTIC_ES_API_KEY` is also set +- **THEN** the system authenticates with the explicit API key + +#### Scenario: Explicit basic authentication is provided + +- **WHEN** the user supplies `--username` and `--password` +- **AND** `ELASTIC_ES_API_KEY` is also set +- **THEN** the system authenticates with the explicit basic credentials diff --git a/openspec/changes/archive/2026-08-26-add-elasticsearch-environment-output/tasks.md b/openspec/changes/archive/2026-08-26-add-elasticsearch-environment-output/tasks.md new file mode 100644 index 0000000..be6073d --- /dev/null +++ b/openspec/changes/archive/2026-08-26-add-elasticsearch-environment-output/tasks.md @@ -0,0 +1,11 @@ +## 1. Environment output implementation + +- [x] 1.1 Reserve `env:/` for environment-backed Elasticsearch output and return `es` and `elasticsearch` to configured-host resolution. +- [x] 1.2 Load missing Elastic connection settings from `.env` without replacing process environment values, and report missing or invalid URLs before ingestion. +- [x] 1.3 Restrict environment API-key fallback to `env:/` output while preserving explicit command-line authentication precedence. + +## 2. Verification and documentation + +- [x] 2.1 Cover environment URI syntax, URL construction, and authentication precedence with unit tests. +- [x] 2.2 Cover process environment precedence, `.env` fallback and parse failure, non-environment scoping, and missing URL failure with CLI tests. +- [x] 2.3 Document `env:/` usage, `.env` lookup, migration from the former schemes, and the new dependency. diff --git a/openspec/specs/elasticsearch-environment-output/spec.md b/openspec/specs/elasticsearch-environment-output/spec.md new file mode 100644 index 0000000..ec5ac1c --- /dev/null +++ b/openspec/specs/elasticsearch-environment-output/spec.md @@ -0,0 +1,99 @@ +## Purpose + +Define how an `env:/` Elasticsearch output resolves connection settings while preserving configured host names and explicit command-line authentication. + +## Requirements + +### Requirement: Environment output uses an explicit URI form + +The system SHALL reserve `env:/` for an Elasticsearch output whose connection settings come from the process environment or `.env`. The URI SHALL contain one slash after `env:` and a non-empty index path. The system SHALL NOT reserve `es` or `elasticsearch` for environment-backed output. + +#### Scenario: Valid environment output is provided + +- **WHEN** the user provides `env:/logs` as the output +- **THEN** the system selects environment-backed Elasticsearch output +- **AND** it uses `logs` as the target index + +#### Scenario: Environment output omits the required index + +- **WHEN** the user provides `env:/` as the output +- **THEN** startup fails with an error that identifies `env:/index` as the required form + +#### Scenario: Environment output uses an authority or omits the slash + +- **WHEN** the user provides `env://logs` or `env:logs` as the output +- **THEN** startup fails with an error that identifies `env:/index` as the required form + +#### Scenario: Former environment scheme is used + +- **WHEN** the user provides an output whose scheme is `es` or `elasticsearch` +- **THEN** the system resolves that scheme as a configured host name +- **AND** it does not read Elastic environment settings for that output + +### Requirement: Environment settings use deterministic precedence + +For an `env:/` output, the system SHALL use values already present in the process environment. For each missing setting, it SHALL search the working directory and its ancestors for the nearest `.env` file and load the setting from that file. A `.env` value SHALL NOT replace a value already present in the process environment. + +#### Scenario: Process environment contains the URL + +- **WHEN** `ELASTIC_ES_URL` is present in the process environment +- **AND** `.env` contains a different `ELASTIC_ES_URL` +- **THEN** the system uses the value from the process environment + +#### Scenario: Dotenv supplies a missing URL + +- **WHEN** `ELASTIC_ES_URL` is absent from the process environment +- **AND** the nearest `.env` file defines `ELASTIC_ES_URL` +- **THEN** the system uses the value from `.env` + +#### Scenario: Dotenv file is malformed + +- **WHEN** the nearest `.env` file cannot be parsed +- **THEN** startup fails with an error that identifies `.env` as unreadable + +#### Scenario: Non-environment output is selected + +- **WHEN** the output does not use the `env` scheme +- **THEN** the system does not load `.env` for Elasticsearch connection settings + +### Requirement: Environment URL is required and valid + +An `env:/` output SHALL require `ELASTIC_ES_URL` after environment and `.env` resolution. The value SHALL be an absolute `http://` or `https://` URL with a host. The system SHALL append the output index to any existing base path and SHALL discard query and fragment components from the configured URL. + +#### Scenario: URL remains unset + +- **WHEN** neither the process environment nor `.env` defines `ELASTIC_ES_URL` +- **THEN** startup fails with an error that names `ELASTIC_ES_URL` + +#### Scenario: URL uses an unsupported or relative form + +- **WHEN** the resolved `ELASTIC_ES_URL` is relative, lacks a host, or uses a scheme other than HTTP or HTTPS +- **THEN** startup fails before sending documents + +#### Scenario: URL contains a base path + +- **WHEN** `ELASTIC_ES_URL` is `https://example.com/elasticsearch/?ignored=true#fragment` +- **AND** the output is `env:/logs` +- **THEN** the Elasticsearch output URL is `https://example.com/elasticsearch/logs` + +### Requirement: Explicit authentication takes precedence + +For an `env:/` output, the system SHALL use `ELASTIC_ES_API_KEY` from the process environment or `.env` when the user supplies no authentication option. An explicit `--apikey` or complete `--username` and `--password` pair SHALL take precedence over `ELASTIC_ES_API_KEY`. + +#### Scenario: Environment API key is the only authentication setting + +- **WHEN** `ELASTIC_ES_API_KEY` resolves from the process environment or `.env` +- **AND** the user supplies no authentication option +- **THEN** the system authenticates with the resolved API key + +#### Scenario: Explicit API key is provided + +- **WHEN** the user supplies `--apikey` +- **AND** `ELASTIC_ES_API_KEY` is also set +- **THEN** the system authenticates with the explicit API key + +#### Scenario: Explicit basic authentication is provided + +- **WHEN** the user supplies `--username` and `--password` +- **AND** `ELASTIC_ES_API_KEY` is also set +- **THEN** the system authenticates with the explicit basic credentials diff --git a/src/main.rs b/src/main.rs index 27a8e07..02eee22 100644 --- a/src/main.rs +++ b/src/main.rs @@ -178,6 +178,12 @@ async fn main() -> ExitCode { template_overwrite, } = args; let output = paths.pop().expect("clap requires at least two paths"); + let environment_output = output + .scheme() + .is_some_and(|scheme| scheme.as_str() == "env"); + if environment_output && let Err(err) = load_dotenv() { + return exit_with_error(err); + } let inputs = paths; let split = match split { Some(path) => match SplitPath::parse(&path) { @@ -194,7 +200,7 @@ async fn main() -> ExitCode { apikey, username.as_deref(), password.as_deref(), - elastic_cli_api_key(), + environment_output.then(environment_api_key).flatten(), ); let auth = match Auth::try_new(apikey, username, password) { Ok(auth) => auth, @@ -234,7 +240,7 @@ async fn main() -> ExitCode { insecure, auth, output, - elastic_cli_url(), + environment_output.then(environment_url).flatten(), action, !uncompressed, elasticsearch_config, @@ -258,7 +264,7 @@ async fn main() -> ExitCode { insecure, auth, output, - elastic_cli_url(), + environment_output.then(environment_url).flatten(), action, !uncompressed, elasticsearch_config, @@ -414,11 +420,19 @@ fn should_discover_input_before_output( inputs.len() > 1 || (explicit_batch_size.is_none() && inputs.iter().all(is_local_file_input)) } -fn elastic_cli_url() -> Option { +fn load_dotenv() -> eyre::Result<()> { + match dotenvy::dotenv() { + Ok(_) => Ok(()), + Err(dotenvy::Error::Io(err)) if err.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(err) => Err(eyre::eyre!("Could not read .env: {err}")), + } +} + +fn environment_url() -> Option { env::var("ELASTIC_ES_URL").ok() } -fn elastic_cli_api_key() -> Option { +fn environment_api_key() -> Option { env::var("ELASTIC_ES_API_KEY").ok() } @@ -426,12 +440,12 @@ fn resolve_api_key( apikey: Option, username: Option<&str>, password: Option<&str>, - elastic_cli_api_key: Option, + environment_api_key: Option, ) -> Option { if apikey.is_some() || username.is_some() || password.is_some() { apikey } else { - elastic_cli_api_key + environment_api_key } } @@ -442,7 +456,7 @@ mod tests { use fluent_uri::UriRef; #[test] - fn elastic_cli_api_key_is_used_without_explicit_authentication() { + fn environment_api_key_is_used_without_explicit_authentication() { assert_eq!( resolve_api_key(None, None, None, Some("context-key".to_string())), Some("context-key".to_string()) @@ -450,7 +464,7 @@ mod tests { } #[test] - fn explicit_authentication_takes_precedence_over_elastic_cli_api_key() { + fn explicit_authentication_takes_precedence_over_environment_api_key() { assert_eq!( resolve_api_key( Some("command-line-key".to_string()), diff --git a/src/output/mod.rs b/src/output/mod.rs index 9adc557..46c5cab 100644 --- a/src/output/mod.rs +++ b/src/output/mod.rs @@ -94,7 +94,7 @@ impl Output { insecure: bool, auth: Auth, uri: UriRef, - elastic_cli_url: Option, + environment_url: Option, action: BulkAction, request_body_compression: bool, elasticsearch_config: ElasticsearchOutputConfig, @@ -102,15 +102,11 @@ impl Output { ) -> Result { log::trace!("{uri:?}"); match uri.scheme() { - Some(scheme) if is_elastic_cli_scheme(scheme.as_str()) => { - let elastic_cli_url = elastic_cli_url.ok_or_else(|| { - eyre!( - "{} outputs require ELASTIC_ES_URL", - elastic_cli_scheme_display(scheme.as_str()) - ) - })?; - let index = elastic_cli_index(&uri)?; - let url = elastic_cli_output_url(&elastic_cli_url, index)?; + Some(scheme) if is_env_scheme(scheme.as_str()) => { + let environment_url = environment_url + .ok_or_else(|| eyre!("env:/index outputs require ELASTIC_ES_URL"))?; + let index = env_index(&uri)?; + let url = environment_output_url(&environment_url, index)?; Self::elasticsearch( insecure, auth, @@ -229,9 +225,9 @@ fn reject_elasticsearch_options(preflight: &OutputPreflightConfig) -> Result<()> Ok(()) } -fn elastic_cli_output_url(elastic_cli_url: &str, index: &str) -> Result { +fn environment_output_url(environment_url: &str, index: &str) -> Result { let mut url = - Url::parse(elastic_cli_url).map_err(|err| eyre!("Invalid ELASTIC_ES_URL: {err}"))?; + Url::parse(environment_url).map_err(|err| eyre!("Invalid ELASTIC_ES_URL: {err}"))?; if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() { return Err(eyre!( "ELASTIC_ES_URL must be an absolute http:// or https:// URL" @@ -245,24 +241,14 @@ fn elastic_cli_output_url(elastic_cli_url: &str, index: &str) -> Result { Ok(url) } -fn is_elastic_cli_scheme(scheme: &str) -> bool { - matches!(scheme, "elasticsearch" | "es") +fn is_env_scheme(scheme: &str) -> bool { + scheme == "env" } -fn elastic_cli_scheme_display(scheme: &str) -> &str { - match scheme { - "elasticsearch" => "elasticsearch:/index", - "es" => "es:/index", - _ => unreachable!("only Elastic CLI output schemes are passed here"), - } -} - -fn elastic_cli_index(uri: &UriRef) -> Result<&str> { +fn env_index(uri: &UriRef) -> Result<&str> { let path = uri.path().as_str(); if uri.authority().is_some() || !path.starts_with('/') || path.len() == 1 { - return Err(eyre!( - "Elastic CLI outputs must use `elasticsearch:/index` or `es:/index`" - )); + return Err(eyre!("environment outputs must use `env:/index`")); } Ok(path.trim_start_matches('/')) } @@ -284,12 +270,12 @@ trait Sender { #[cfg(test)] mod tests { - use super::{elastic_cli_index, elastic_cli_output_url, is_elastic_cli_scheme}; + use super::{env_index, environment_output_url, is_env_scheme}; use fluent_uri::UriRef; #[test] - fn elastic_cli_url_appends_index_to_base_path() { - let url = elastic_cli_output_url( + fn environment_url_appends_index_to_base_path() { + let url = environment_output_url( "https://example.com/elasticsearch/?ignored=true#fragment", "logs-2026", ) @@ -299,28 +285,29 @@ mod tests { } #[test] - fn elastic_cli_url_requires_an_absolute_http_url() { - let err = elastic_cli_output_url("file:///tmp/elasticsearch", "logs").unwrap_err(); + fn environment_url_requires_an_absolute_http_url() { + let err = environment_output_url("file:///tmp/elasticsearch", "logs").unwrap_err(); assert!(err.to_string().contains("http:// or https://")); } #[test] - fn elastic_cli_schemes_are_reserved_for_context_outputs() { - assert!(is_elastic_cli_scheme("elasticsearch")); - assert!(is_elastic_cli_scheme("es")); - assert!(!is_elastic_cli_scheme("production")); + fn env_scheme_is_reserved_for_environment_outputs() { + assert!(is_env_scheme("env")); + assert!(!is_env_scheme("elasticsearch")); + assert!(!is_env_scheme("es")); + assert!(!is_env_scheme("production")); } #[test] - fn elastic_cli_index_requires_a_single_slash_after_scheme() { - let es = UriRef::parse("es:/logs-2026".to_string()).unwrap(); - assert_eq!(elastic_cli_index(&es).unwrap(), "logs-2026"); + fn env_index_requires_a_single_slash_after_scheme() { + let env = UriRef::parse("env:/logs-2026".to_string()).unwrap(); + assert_eq!(env_index(&env).unwrap(), "logs-2026"); - let missing_slash = UriRef::parse("es:logs-2026".to_string()).unwrap(); - assert!(elastic_cli_index(&missing_slash).is_err()); + let missing_slash = UriRef::parse("env:logs-2026".to_string()).unwrap(); + assert!(env_index(&missing_slash).is_err()); - let authority = UriRef::parse("es://logs-2026".to_string()).unwrap(); - assert!(elastic_cli_index(&authority).is_err()); + let authority = UriRef::parse("env://logs-2026".to_string()).unwrap(); + assert!(env_index(&authority).is_err()); } } diff --git a/tests/env_output.rs b/tests/env_output.rs new file mode 100644 index 0000000..a12ceaa --- /dev/null +++ b/tests/env_output.rs @@ -0,0 +1,105 @@ +use std::{fs, process::Command}; + +fn workspace() -> tempfile::TempDir { + let dir = tempfile::tempdir().expect("create temporary workspace"); + fs::write(dir.path().join("docs.ndjson"), "{\"message\":\"hello\"}\n") + .expect("write input fixture"); + dir +} + +fn run_espipe(dir: &tempfile::TempDir, environment_url: Option<&str>) -> std::process::Output { + run_espipe_to(dir, "env:/logs", environment_url) +} + +fn run_espipe_to( + dir: &tempfile::TempDir, + output: &str, + environment_url: Option<&str>, +) -> std::process::Output { + let mut command = Command::new(env!("CARGO_BIN_EXE_espipe")); + command + .current_dir(dir.path()) + .env_remove("ELASTIC_ES_URL") + .env_remove("ELASTIC_ES_API_KEY") + .args(["docs.ndjson", output]); + if let Some(url) = environment_url { + command.env("ELASTIC_ES_URL", url); + } + command.output().expect("run espipe") +} + +#[test] +fn env_output_fails_when_url_is_absent_from_environment_and_dotenv() { + let dir = workspace(); + let output = run_espipe(&dir, None); + + assert!(!output.status.success()); + assert_eq!( + String::from_utf8_lossy(&output.stderr), + "env:/index outputs require ELASTIC_ES_URL\n" + ); +} + +#[test] +fn env_output_reads_url_from_dotenv() { + let dir = workspace(); + fs::write( + dir.path().join(".env"), + "ELASTIC_ES_URL=file:///dotenv-value\n", + ) + .expect("write .env"); + + let output = run_espipe(&dir, None); + + assert!(!output.status.success()); + assert!( + String::from_utf8_lossy(&output.stderr) + .contains("ELASTIC_ES_URL must be an absolute http:// or https:// URL") + ); +} + +#[test] +fn process_environment_takes_precedence_over_dotenv() { + let dir = workspace(); + fs::write( + dir.path().join(".env"), + "ELASTIC_ES_URL=https://127.0.0.1:1\n", + ) + .expect("write .env"); + + let output = run_espipe(&dir, Some("file:///process-environment-value")); + + assert!(!output.status.success()); + assert!( + String::from_utf8_lossy(&output.stderr) + .contains("ELASTIC_ES_URL must be an absolute http:// or https:// URL") + ); +} + +#[test] +fn malformed_dotenv_fails_environment_output() { + let dir = workspace(); + fs::write(dir.path().join(".env"), "ELASTIC_ES_URL='unterminated\n") + .expect("write malformed .env"); + + let output = run_espipe(&dir, None); + + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("Could not read .env")); +} + +#[test] +fn non_environment_output_does_not_load_dotenv() { + let dir = workspace(); + fs::write(dir.path().join(".env"), "ELASTIC_ES_URL='unterminated\n") + .expect("write malformed .env"); + + let output = run_espipe_to(&dir, "output.ndjson", None); + + assert!(output.status.success()); + assert!( + fs::read_to_string(dir.path().join("output.ndjson")) + .expect("read output") + .contains("\"message\":\"hello\"") + ); +} diff --git a/tests/index_template.rs b/tests/index_template.rs index 0943fbb..99fbda7 100644 --- a/tests/index_template.rs +++ b/tests/index_template.rs @@ -529,10 +529,20 @@ fn cli_upsert_reuses_file_ids_after_content_changes_and_file_additions() { .map(|line| serde_json::from_str(line).unwrap()) .collect(); assert_eq!(second_lines.len(), 4); - assert_eq!(second_lines[0]["update"]["_id"], first_id); + let (operations, remainder) = second_lines.as_chunks::<2>(); + assert!(remainder.is_empty()); + let first_operation = operations + .iter() + .find(|operation| operation[1]["doc"]["origin"]["filename"] == "first.md") + .expect("first.md bulk operation"); + let second_operation = operations + .iter() + .find(|operation| operation[1]["doc"]["origin"]["filename"] == "second.md") + .expect("second.md bulk operation"); + assert_eq!(first_operation[0]["update"]["_id"], first_id); assert_ne!( - second_lines[0]["update"]["_id"], - second_lines[2]["update"]["_id"] + first_operation[0]["update"]["_id"], + second_operation[0]["update"]["_id"] ); } From 38f49588701f2595baf2acd281eee33f73d37665 Mon Sep 17 00:00:00 2001 From: Ryan Eno Date: Wed, 26 Aug 2026 20:23:53 -0700 Subject: [PATCH 2/2] fix: validate environment target before dotenv --- CHANGELOG.md | 2 +- src/main.rs | 7 ++++--- src/output/mod.rs | 27 ++++++++++++++++++++++++++- tests/env_output.rs | 19 +++++++++++++++++++ 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9336c5..81b1f1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added `.env` fallback for missing `ELASTIC_ES_URL` and `ELASTIC_ES_API_KEY` environment settings. +- Added `.env` fallback for missing `ELASTIC_ES_URL` and `ELASTIC_ES_API_KEY` settings used by `env:/` outputs. ### Changed diff --git a/src/main.rs b/src/main.rs index 02eee22..f079292 100644 --- a/src/main.rs +++ b/src/main.rs @@ -178,9 +178,10 @@ async fn main() -> ExitCode { template_overwrite, } = args; let output = paths.pop().expect("clap requires at least two paths"); - let environment_output = output - .scheme() - .is_some_and(|scheme| scheme.as_str() == "env"); + let environment_output = match Output::validate_environment_target(&output) { + Ok(environment_output) => environment_output, + Err(err) => return exit_with_error(err), + }; if environment_output && let Err(err) = load_dotenv() { return exit_with_error(err); } diff --git a/src/output/mod.rs b/src/output/mod.rs index 46c5cab..e3ca02f 100644 --- a/src/output/mod.rs +++ b/src/output/mod.rs @@ -80,6 +80,17 @@ impl OutputPreflightConfig { } impl Output { + pub fn validate_environment_target(uri: &UriRef) -> Result { + if uri + .scheme() + .is_some_and(|scheme| is_env_scheme(scheme.as_str())) + { + env_index(uri)?; + return Ok(true); + } + Ok(false) + } + pub fn validate_preflight_target( uri: &UriRef, preflight: &OutputPreflightConfig, @@ -270,7 +281,7 @@ trait Sender { #[cfg(test)] mod tests { - use super::{env_index, environment_output_url, is_env_scheme}; + use super::{Output, env_index, environment_output_url, is_env_scheme}; use fluent_uri::UriRef; #[test] @@ -310,4 +321,18 @@ mod tests { let authority = UriRef::parse("env://logs-2026".to_string()).unwrap(); assert!(env_index(&authority).is_err()); } + + #[test] + fn environment_target_validation_rejects_invalid_uri_forms() { + let valid = UriRef::parse("env:/logs-2026".to_string()).unwrap(); + assert!(Output::validate_environment_target(&valid).unwrap()); + + for invalid in ["env:/", "env:logs-2026", "env://logs-2026"] { + let uri = UriRef::parse(invalid.to_string()).unwrap(); + assert!(Output::validate_environment_target(&uri).is_err()); + } + + let direct = UriRef::parse("https://example.com/logs".to_string()).unwrap(); + assert!(!Output::validate_environment_target(&direct).unwrap()); + } } diff --git a/tests/env_output.rs b/tests/env_output.rs index a12ceaa..ed17bd5 100644 --- a/tests/env_output.rs +++ b/tests/env_output.rs @@ -88,6 +88,25 @@ fn malformed_dotenv_fails_environment_output() { assert!(String::from_utf8_lossy(&output.stderr).contains("Could not read .env")); } +#[test] +fn invalid_environment_uri_is_rejected_before_dotenv_is_loaded() { + let dir = workspace(); + fs::write(dir.path().join(".env"), "ELASTIC_ES_URL='unterminated\n") + .expect("write malformed .env"); + + for target in ["env:/", "env:logs", "env://logs"] { + let output = run_espipe_to(&dir, target, None); + let stderr = String::from_utf8_lossy(&output.stderr); + + assert!(!output.status.success()); + assert!( + stderr.contains("environment outputs must use `env:/index`"), + "stderr for {target}: {stderr}" + ); + assert!(!stderr.contains("Could not read .env")); + } +} + #[test] fn non_environment_output_does_not_load_dotenv() { let dir = workspace();