From 05fe2e6a71c82dba188e3cba08d8643706e42ef9 Mon Sep 17 00:00:00 2001 From: wangxianghu Date: Sat, 5 Sep 2026 03:09:13 +0400 Subject: [PATCH] [AMORO-4359][AMS] Support AWS Secrets Manager as a ConfigShade --- LICENSE-binary | 1 + amoro-ams/pom.xml | 16 ++ .../config/shade/AwsSecretsManagerClient.java | 42 ++++ .../shade/AwsSecretsManagerConfigShade.java | 186 +++++++++++++++ .../shade/DefaultAwsSecretsManagerClient.java | 112 +++++++++ .../server/config/shade/SecretReference.java | 136 +++++++++++ .../org.apache.amoro.config.shade.ConfigShade | 19 ++ .../TestAwsSecretsManagerConfigShade.java | 222 ++++++++++++++++++ .../config/shade/TestSecretReference.java | 120 ++++++++++ dev/deps/dependencies-hadoop-2-spark-3.3 | 1 + dev/deps/dependencies-hadoop-3-spark-3.5 | 1 + pom.xml | 6 + 12 files changed, 862 insertions(+) create mode 100644 amoro-ams/src/main/java/org/apache/amoro/server/config/shade/AwsSecretsManagerClient.java create mode 100644 amoro-ams/src/main/java/org/apache/amoro/server/config/shade/AwsSecretsManagerConfigShade.java create mode 100644 amoro-ams/src/main/java/org/apache/amoro/server/config/shade/DefaultAwsSecretsManagerClient.java create mode 100644 amoro-ams/src/main/java/org/apache/amoro/server/config/shade/SecretReference.java create mode 100644 amoro-ams/src/main/resources/META-INF/services/org.apache.amoro.config.shade.ConfigShade create mode 100644 amoro-ams/src/test/java/org/apache/amoro/server/config/shade/TestAwsSecretsManagerConfigShade.java create mode 100644 amoro-ams/src/test/java/org/apache/amoro/server/config/shade/TestSecretReference.java diff --git a/LICENSE-binary b/LICENSE-binary index 5a57a2f3ac..72192356f0 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -328,6 +328,7 @@ software.amazon.awssdk:protocol-core software.amazon.awssdk:regions software.amazon.awssdk:s3 software.amazon.awssdk:sdk-core +software.amazon.awssdk:secretsmanager software.amazon.awssdk:sts software.amazon.awssdk:third-party-jackson-core software.amazon.awssdk:url-connection-client diff --git a/amoro-ams/pom.xml b/amoro-ams/pom.xml index 47d4a48232..8cfc07e404 100644 --- a/amoro-ams/pom.xml +++ b/amoro-ams/pom.xml @@ -336,6 +336,22 @@ url-connection-client + + + software.amazon.awssdk + secretsmanager + + + software.amazon.awssdk + netty-nio-client + + + software.amazon.awssdk + apache-client + + + + software.amazon.awssdk s3-transfer-manager diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/AwsSecretsManagerClient.java b/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/AwsSecretsManagerClient.java new file mode 100644 index 0000000000..2df819c968 --- /dev/null +++ b/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/AwsSecretsManagerClient.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.server.config.shade; + +/** + * A minimal abstraction over AWS Secrets Manager. It only exposes a single method for fetching a + * secret string, which makes it easy to stub in unit tests. + * + *

Implementations are expected to cache within the process — AMS only calls {@link + * #getSecretString(String)} during startup, but the same secret may be referenced in several places + * (for example, the username/password of the same DB living in the same JSON). + */ +interface AwsSecretsManagerClient extends AutoCloseable { + + /** + * Fetches the {@code SecretString} content of the secret. + * + * @param secretId the secret name or ARN + * @return the plaintext of the secret (either a plain string or JSON) + * @throws RuntimeException if the fetch fails; the caller decides whether to fail-fast + */ + String getSecretString(String secretId); + + @Override + void close(); +} diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/AwsSecretsManagerConfigShade.java b/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/AwsSecretsManagerConfigShade.java new file mode 100644 index 0000000000..3297160ab4 --- /dev/null +++ b/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/AwsSecretsManagerConfigShade.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.server.config.shade; + +import org.apache.amoro.config.Configurations; +import org.apache.amoro.config.shade.ConfigShade; +import org.apache.amoro.shade.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.amoro.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Objects; +import java.util.function.Function; + +/** + * Delegates AMS's sensitive configs (db.username / db.password, etc.) to AWS Secrets Manager + * through Amoro's {@link ConfigShade} SPI. + * + *

Usage (in {@code $AMORO_HOME/conf/config.yaml}): + * + * 

{@code
+ * ams:
+ *   shade:
+ *     identifier: aws-sm
+ *     sensitive-keywords: database.username;database.password
+ *   database:
+ *     # both point to the same JSON secret; '#' selects the value out of it
+ *     username: arn:aws:secretsmanager:ap-northeast-1:123456789012:secret:prod-amoro-db-mOhyOp#db.username
+ *     password: arn:aws:secretsmanager:ap-northeast-1:123456789012:secret:prod-amoro-db-mOhyOp#db.password
+ * }
+ * + *

The trailing {@code -mOhyOp} is the six random characters AWS appends to every secret ARN; use + * the full ARN as shown. The optional {@code #} suffix selects one field when the secret + * value is JSON, e.g. {@code {"db.username": "amoro", "db.password": "..."}} — the field name is + * whatever key you stored, taken as-is (dots are part of the key, not a nested path). Omit the + * suffix when the secret value is the plaintext itself. + * + *

No explicit region configuration is required — it is resolved from the ARN. All secrets shaded + * for one AMS instance are expected to live in the same region, so a single client is created + * (lazily, memoized within the process); a reference resolving to a different region is treated as + * a misconfiguration and fails fast. + * + *

AMS startup flow: {@code AmoroServiceContainer.initServiceConfig} → {@code + * ConfigShadeUtils.decryptConfig} → find this SPI instance with identifier=aws-sm → call {@link + * #decrypt(String)} for each sensitive key. + * + *

Failure strategy: always fail-fast (malformed ARN, fetch failure, missing JSON field) so that + * AMS startup blows up immediately, instead of continuing with a wrong password — errors that + * surface after the DB receives a wrong password are far from the root cause. + */ +public class AwsSecretsManagerConfigShade implements ConfigShade { + + private static final Logger LOG = LoggerFactory.getLogger(AwsSecretsManagerConfigShade.class); + + public static final String IDENTIFIER = "aws-sm"; + + private final ObjectMapper jsonMapper; + + /** + * A single client, lazily initialized on the first decrypt() call. All secrets shaded for one AMS + * instance (admin-password, database.password, ...) live in the same region, so one client is + * enough. Guarded by {@code this} for the double-checked lazy init; the region it was bound to is + * validated against every subsequent reference so that a misconfigured cross-region ARN fails + * fast instead of being fetched with the wrong client. + */ + private volatile AwsSecretsManagerClient client; + + private volatile String boundRegion; + + /** + * The factory that creates a client; {@link DefaultAwsSecretsManagerClient#create} for + * production, a mock injected in unit tests. + */ + private final Function clientFactory; + + /** No-arg SPI constructor. */ + public AwsSecretsManagerConfigShade() { + this(new ObjectMapper(), DefaultAwsSecretsManagerClient::create); + } + + /** For injecting a mock client factory in unit tests. */ + AwsSecretsManagerConfigShade( + ObjectMapper jsonMapper, Function clientFactory) { + this.jsonMapper = Objects.requireNonNull(jsonMapper, "jsonMapper"); + this.clientFactory = Objects.requireNonNull(clientFactory, "clientFactory"); + } + + @Override + public String getIdentifier() { + return IDENTIFIER; + } + + @Override + public void initialize(Configurations serviceConfig) { + // No SPI-level config to read — region is resolved per ARN; credentials use the AWS default + // provider chain. + LOG.info("AwsSecretsManagerConfigShade initialized (region resolved per-secret from ARN)"); + } + + @Override + public String decrypt(String content) { + SecretReference ref = SecretReference.parse(content); + String raw = getOrCreateClient(ref.region()).getSecretString(ref.secretArn()); + return ref.jsonKey().map(key -> extractJsonField(raw, key, ref.secretArn())).orElse(raw); + } + + /** + * Returns the shared client, creating it on the first call. All shaded secrets are expected to + * live in a single region; if a later reference resolves to a different region it is almost + * certainly a misconfiguration, so we fail fast rather than fetch it with the wrong client. + */ + private AwsSecretsManagerClient getOrCreateClient(String region) { + AwsSecretsManagerClient existing = client; + if (existing == null) { + synchronized (this) { + existing = client; + if (existing == null) { + existing = clientFactory.apply(region); + boundRegion = region; + client = existing; + } + } + } + if (!Objects.equals(region, boundRegion)) { + throw new IllegalStateException( + "all AWS Secrets Manager references must be in the same region; expected '" + + boundRegion + + "' but got '" + + region + + "'"); + } + return existing; + } + + private String extractJsonField(String json, String key, String secretArn) { + JsonNode root; + try { + root = jsonMapper.readTree(json); + } catch (IOException e) { + // Do not log the raw JSON; it may contain other sensitive fields. + throw new IllegalStateException( + "secret '" + secretArn + "' is not valid JSON; cannot extract field '" + key + "'", e); + } + if (root == null || !root.isObject()) { + throw new IllegalStateException( + "secret '" + secretArn + "' is not a JSON object; cannot extract field '" + key + "'"); + } + JsonNode value = root.get(key); + if (value == null || value.isNull()) { + throw new IllegalStateException("secret '" + secretArn + "' has no field '" + key + "'"); + } + // A config value must be a scalar. For an object/array node asText() silently returns "", which + // would then be handed to the DB as e.g. an empty password — the far-from-root-cause failure + // this class fails fast to avoid. So reject non-scalar values explicitly. + if (!value.isValueNode()) { + throw new IllegalStateException( + "secret '" + + secretArn + + "' field '" + + key + + "' must be a scalar value, but is a " + + value.getNodeType() + + "; a structured JSON value cannot be used as a config value"); + } + // asText() returns strings as-is and converts numbers/booleans to strings — a good fit for the + // password scenario. + return value.asText(); + } +} diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/DefaultAwsSecretsManagerClient.java b/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/DefaultAwsSecretsManagerClient.java new file mode 100644 index 0000000000..b8a6017498 --- /dev/null +++ b/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/DefaultAwsSecretsManagerClient.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.server.config.shade; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse; + +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * The default {@link AwsSecretsManagerClient} implementation, bound to a single region. + * + *

+ * + *

On failure it throws {@link IllegalStateException} to make AMS startup fail-fast, avoiding + * starting the service with a wrong password. + */ +final class DefaultAwsSecretsManagerClient implements AwsSecretsManagerClient { + + private static final Logger LOG = LoggerFactory.getLogger(DefaultAwsSecretsManagerClient.class); + + private final SecretsManagerClient delegate; + private final String region; + private final ConcurrentMap cache = new ConcurrentHashMap<>(); + + private DefaultAwsSecretsManagerClient(SecretsManagerClient delegate, String region) { + this.delegate = Objects.requireNonNull(delegate, "delegate"); + this.region = region; + } + + static DefaultAwsSecretsManagerClient create(String region) { + Objects.requireNonNull(region, "region"); + SecretsManagerClient sdkClient = + SecretsManagerClient.builder() + .region(Region.of(region)) + .credentialsProvider(DefaultCredentialsProvider.create()) + .httpClient(UrlConnectionHttpClient.create()) + .build(); + LOG.info("AWS Secrets Manager client initialized (region={})", region); + return new DefaultAwsSecretsManagerClient(sdkClient, region); + } + + @Override + public String getSecretString(String secretId) { + return cache.computeIfAbsent(secretId, this::fetch); + } + + private String fetch(String secretId) { + try { + GetSecretValueResponse response = + delegate.getSecretValue(GetSecretValueRequest.builder().secretId(secretId).build()); + String value = response.secretString(); + if (value == null) { + // Binary secrets are rarely used for passwords; fail clearly if one is encountered. + throw new IllegalStateException( + "secret '" + secretId + "' has no SecretString (binary secrets are not supported)"); + } + LOG.info("Fetched secret '{}' from AWS Secrets Manager (region={})", secretId, region); + return value; + } catch (RuntimeException e) { + // Mask the secret value itself, but keep secretId / region for operational troubleshooting. + throw new IllegalStateException( + "Failed to fetch secret '" + + secretId + + "' from AWS Secrets Manager (region=" + + region + + "): " + + e.getMessage(), + e); + } + } + + @Override + public void close() { + try { + delegate.close(); + } catch (RuntimeException e) { + LOG.warn("Error closing SecretsManagerClient (region={})", region, e); + } + } +} diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/SecretReference.java b/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/SecretReference.java new file mode 100644 index 0000000000..cf40454d65 --- /dev/null +++ b/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/SecretReference.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.server.config.shade; + +import java.util.Objects; +import java.util.Optional; + +/** + * Parses an AWS Secrets Manager reference written in {@code config.yaml}. + * + *

Only a full ARN is accepted (short names are not supported); the region is extracted directly + * from the ARN to avoid additional configuration. + * + *

Syntax: + * + * 

{@code
+ * arn:aws:secretsmanager:::secret:[-]
+ * arn:aws:secretsmanager:::secret:[-]#
+ * }
+ * + *

Examples: + * + *

+ * + *

The delimiter is fixed to {@code #} to avoid the {@code :} that fills ARNs and the {@code /} + * commonly found in secret names. + */ +final class SecretReference { + + private static final char KEY_DELIMITER = '#'; + + /** ARN fixed format: {@code arn:aws:secretsmanager:REGION:ACCOUNT:secret:NAME} — 7 parts. */ + private static final int ARN_PARTS = 7; + + private static final String ARN_PREFIX = "arn:"; + private static final String ARN_SERVICE = "secretsmanager"; + private static final String ARN_RESOURCE_TYPE = "secret"; + + private final String secretArn; + private final String region; + private final String jsonKey; // nullable + + private SecretReference(String secretArn, String region, String jsonKey) { + this.secretArn = secretArn; + this.region = region; + this.jsonKey = jsonKey; + } + + static SecretReference parse(String content) { + Objects.requireNonNull(content, "secret reference content is null"); + String trimmed = content.trim(); + if (trimmed.isEmpty()) { + throw new IllegalArgumentException("secret reference is blank"); + } + + // Split off the optional '#jsonKey' suffix first. + String arnPart; + String jsonKey; + int hashIdx = trimmed.lastIndexOf(KEY_DELIMITER); + if (hashIdx < 0) { + arnPart = trimmed; + jsonKey = null; + } else { + arnPart = trimmed.substring(0, hashIdx).trim(); + String tail = trimmed.substring(hashIdx + 1).trim(); + jsonKey = tail.isEmpty() ? null : tail; + } + + validateArn(arnPart); + String region = extractRegion(arnPart); + return new SecretReference(arnPart, region, jsonKey); + } + + private static void validateArn(String arn) { + if (!arn.startsWith(ARN_PREFIX)) { + throw new IllegalArgumentException( + "secret reference must be a full ARN (arn:aws:secretsmanager:::secret:)"); + } + // limit=ARN_PARTS ensures that an illegal ':' inside the secret name is not over-split. + String[] parts = arn.split(":", ARN_PARTS); + if (parts.length != ARN_PARTS + || !ARN_SERVICE.equals(parts[2]) + || !ARN_RESOURCE_TYPE.equals(parts[5]) + || parts[3].isEmpty() + || parts[6].isEmpty()) { + throw new IllegalArgumentException( + "malformed AWS Secrets Manager ARN (expected arn:aws:secretsmanager:::secret:)"); + } + } + + private static String extractRegion(String arn) { + // At this point the ARN has passed validateArn and is guaranteed to have 7 parts. + return arn.split(":", ARN_PARTS)[3]; + } + + /** The full ARN — pass it directly to {@code GetSecretValueRequest#secretId}. */ + String secretArn() { + return secretArn; + } + + /** The region extracted from the ARN; used to select/create the {@code SecretsManagerClient}. */ + String region() { + return region; + } + + Optional jsonKey() { + return Optional.ofNullable(jsonKey); + } + + @Override + public String toString() { + return jsonKey == null ? secretArn : secretArn + KEY_DELIMITER + jsonKey; + } +} diff --git a/amoro-ams/src/main/resources/META-INF/services/org.apache.amoro.config.shade.ConfigShade b/amoro-ams/src/main/resources/META-INF/services/org.apache.amoro.config.shade.ConfigShade new file mode 100644 index 0000000000..8e58374e38 --- /dev/null +++ b/amoro-ams/src/main/resources/META-INF/services/org.apache.amoro.config.shade.ConfigShade @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# * +# http://www.apache.org/licenses/LICENSE-2.0 +# * +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.amoro.server.config.shade.AwsSecretsManagerConfigShade diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/config/shade/TestAwsSecretsManagerConfigShade.java b/amoro-ams/src/test/java/org/apache/amoro/server/config/shade/TestAwsSecretsManagerConfigShade.java new file mode 100644 index 0000000000..67a68deb0f --- /dev/null +++ b/amoro-ams/src/test/java/org/apache/amoro/server/config/shade/TestAwsSecretsManagerConfigShade.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.server.config.shade; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.apache.amoro.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; + +/** Unit tests for {@link AwsSecretsManagerConfigShade} decryption logic with a mocked client. */ +public class TestAwsSecretsManagerConfigShade { + + /** + * The common case: one secret holds a JSON blob with many key-value pairs, and each config value + * selects a field with {@code #}. Most tests use this ARN. + */ + private static final String DB_SECRET_ARN = + "arn:aws:secretsmanager:ap-northeast-1:123456789012:secret:prod-amoro-db-mOhyOp"; + + /** + * A secret whose value is a single plaintext string (no {@code #}); exercises the non-JSON + * branch. + */ + private static final String PLAINTEXT_SECRET_ARN = + "arn:aws:secretsmanager:ap-northeast-1:123456789012:secret:prod-amoro-admin-AbCdEf"; + + /** A secret in a DIFFERENT region, used only to drive the cross-region fail-fast test. */ + private static final String OTHER_REGION_SECRET_ARN = + "arn:aws:secretsmanager:us-east-1:123456789012:secret:prod-amoro-other-XyZ123"; + + @Test + public void testIdentifier() { + assertEquals("aws-sm", new AwsSecretsManagerConfigShade().getIdentifier()); + } + + @Test + public void testDecryptPlainSecret() { + AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class); + when(client.getSecretString(PLAINTEXT_SECRET_ARN)).thenReturn("super-secret-password"); + + AwsSecretsManagerConfigShade shade = + new AwsSecretsManagerConfigShade(new ObjectMapper(), region -> client); + + assertEquals("super-secret-password", shade.decrypt(PLAINTEXT_SECRET_ARN)); + verify(client).getSecretString(PLAINTEXT_SECRET_ARN); + } + + @Test + public void testDecryptJsonField() { + // Username and password live in the same JSON secret, selected by dotted field names. + AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class); + when(client.getSecretString(DB_SECRET_ARN)) + .thenReturn("{\"db.username\":\"amoro\",\"db.password\":\"p@ss\"}"); + + AwsSecretsManagerConfigShade shade = + new AwsSecretsManagerConfigShade(new ObjectMapper(), region -> client); + + assertEquals("amoro", shade.decrypt(DB_SECRET_ARN + "#db.username")); + assertEquals("p@ss", shade.decrypt(DB_SECRET_ARN + "#db.password")); + } + + @Test + public void testDottedFieldNameIsTakenAsIsNotNestedPath() { + // A '.' in the field name is part of the key, not a nested-path separator: a flat key with dots + // resolves, while the same dots interpreted as nesting must not. + AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class); + when(client.getSecretString(DB_SECRET_ARN)) + .thenReturn("{\"db.password\":\"flat\",\"db\":{\"password\":\"nested\"}}"); + + AwsSecretsManagerConfigShade shade = + new AwsSecretsManagerConfigShade(new ObjectMapper(), region -> client); + + // Must match the flat key literally, never drill into the nested object. + assertEquals("flat", shade.decrypt(DB_SECRET_ARN + "#db.password")); + } + + @Test + public void testDecryptJsonFieldConvertsNonStringValue() { + AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class); + when(client.getSecretString(DB_SECRET_ARN)).thenReturn("{\"db.port\":5432}"); + + AwsSecretsManagerConfigShade shade = + new AwsSecretsManagerConfigShade(new ObjectMapper(), region -> client); + + assertEquals("5432", shade.decrypt(DB_SECRET_ARN + "#db.port")); + } + + @Test + public void testClientIsMemoized() { + AtomicInteger created = new AtomicInteger(); + AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class); + when(client.getSecretString(DB_SECRET_ARN)).thenReturn("{\"db.password\":\"p@ss\"}"); + Function factory = + region -> { + created.incrementAndGet(); + return client; + }; + + AwsSecretsManagerConfigShade shade = + new AwsSecretsManagerConfigShade(new ObjectMapper(), factory); + + shade.decrypt(DB_SECRET_ARN + "#db.password"); + shade.decrypt(DB_SECRET_ARN + "#db.password"); + // Multiple decrypt calls must reuse a single lazily-created client. + assertEquals(1, created.get()); + verify(client, times(2)).getSecretString(DB_SECRET_ARN); + } + + @Test + public void testCrossRegionReferenceFailsFast() { + AtomicInteger created = new AtomicInteger(); + AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class); + when(client.getSecretString(PLAINTEXT_SECRET_ARN)).thenReturn("v1"); + Function factory = + region -> { + created.incrementAndGet(); + return client; + }; + + AwsSecretsManagerConfigShade shade = + new AwsSecretsManagerConfigShade(new ObjectMapper(), factory); + + shade.decrypt(PLAINTEXT_SECRET_ARN); // ap-northeast-1 binds the client + // A second reference in a different region is treated as a misconfiguration. + assertThrows( + IllegalStateException.class, () -> shade.decrypt(OTHER_REGION_SECRET_ARN)); // us-east-1 + // The client is created only once and never rebound to the second region. + assertEquals(1, created.get()); + } + + @Test + public void testMissingJsonFieldFailsFast() { + AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class); + when(client.getSecretString(DB_SECRET_ARN)).thenReturn("{\"db.username\":\"amoro\"}"); + + AwsSecretsManagerConfigShade shade = + new AwsSecretsManagerConfigShade(new ObjectMapper(), region -> client); + + assertThrows(IllegalStateException.class, () -> shade.decrypt(DB_SECRET_ARN + "#db.password")); + } + + @Test + public void testJsonFieldPointingToObjectFailsFast() { + // '#db' selects an object node. Jackson's asText() would silently return "" for it, so without + // the scalar check the DB would receive an empty value; the shade must fail fast instead. + AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class); + when(client.getSecretString(DB_SECRET_ARN)) + .thenReturn("{\"db\":{\"username\":\"amoro\",\"password\":\"p@ss\"}}"); + + AwsSecretsManagerConfigShade shade = + new AwsSecretsManagerConfigShade(new ObjectMapper(), region -> client); + + assertThrows(IllegalStateException.class, () -> shade.decrypt(DB_SECRET_ARN + "#db")); + } + + @Test + public void testJsonFieldPointingToArrayFailsFast() { + // Same trap as an object node: '#hosts' selects an array, for which asText() returns "". + AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class); + when(client.getSecretString(DB_SECRET_ARN)).thenReturn("{\"hosts\":[\"a\",\"b\"]}"); + + AwsSecretsManagerConfigShade shade = + new AwsSecretsManagerConfigShade(new ObjectMapper(), region -> client); + + assertThrows(IllegalStateException.class, () -> shade.decrypt(DB_SECRET_ARN + "#hosts")); + } + + @Test + public void testInvalidJsonFailsFast() { + AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class); + when(client.getSecretString(DB_SECRET_ARN)).thenReturn("not-a-json"); + + AwsSecretsManagerConfigShade shade = + new AwsSecretsManagerConfigShade(new ObjectMapper(), region -> client); + + assertThrows(IllegalStateException.class, () -> shade.decrypt(DB_SECRET_ARN + "#db.password")); + } + + @Test + public void testNonObjectJsonFailsFast() { + AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class); + when(client.getSecretString(DB_SECRET_ARN)).thenReturn("[1,2,3]"); + + AwsSecretsManagerConfigShade shade = + new AwsSecretsManagerConfigShade(new ObjectMapper(), region -> client); + + assertThrows(IllegalStateException.class, () -> shade.decrypt(DB_SECRET_ARN + "#db.password")); + } + + @Test + public void testMalformedReferenceFailsFast() { + AwsSecretsManagerConfigShade shade = + new AwsSecretsManagerConfigShade( + new ObjectMapper(), region -> mock(AwsSecretsManagerClient.class)); + + assertThrows(IllegalArgumentException.class, () -> shade.decrypt("not-an-arn")); + } +} diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/config/shade/TestSecretReference.java b/amoro-ams/src/test/java/org/apache/amoro/server/config/shade/TestSecretReference.java new file mode 100644 index 0000000000..6c7382d402 --- /dev/null +++ b/amoro-ams/src/test/java/org/apache/amoro/server/config/shade/TestSecretReference.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.server.config.shade; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +/** Unit tests for {@link SecretReference} ARN parsing and validation. */ +public class TestSecretReference { + + private static final String DB_SECRET_ARN = + "arn:aws:secretsmanager:ap-northeast-1:123456789012:secret:prod-amoro-db-mOhyOp"; + + @Test + public void testParsePlainArn() { + SecretReference ref = SecretReference.parse(DB_SECRET_ARN); + assertEquals(DB_SECRET_ARN, ref.secretArn()); + assertEquals("ap-northeast-1", ref.region()); + assertFalse(ref.jsonKey().isPresent()); + } + + @Test + public void testParseArnWithJsonKey() { + SecretReference ref = SecretReference.parse(DB_SECRET_ARN + "#db.password"); + // The '#jsonKey' suffix must be stripped from the ARN passed to the SDK. + assertEquals(DB_SECRET_ARN, ref.secretArn()); + assertEquals("ap-northeast-1", ref.region()); + assertTrue(ref.jsonKey().isPresent()); + // A dotted field name is kept intact: '#' is split on lastIndexOf, dots are part of the key. + assertEquals("db.password", ref.jsonKey().get()); + } + + @Test + public void testParseTrimsSurroundingWhitespace() { + SecretReference ref = SecretReference.parse(" " + DB_SECRET_ARN + " # db.password "); + assertEquals(DB_SECRET_ARN, ref.secretArn()); + assertEquals("db.password", ref.jsonKey().get()); + } + + @Test + public void testEmptyJsonKeyTreatedAsAbsent() { + // A trailing '#' with no key should behave like a plain ARN reference. + SecretReference ref = SecretReference.parse(DB_SECRET_ARN + "#"); + assertEquals(DB_SECRET_ARN, ref.secretArn()); + assertFalse(ref.jsonKey().isPresent()); + } + + @Test + public void testToStringRoundTrip() { + assertEquals(DB_SECRET_ARN, SecretReference.parse(DB_SECRET_ARN).toString()); + assertEquals( + DB_SECRET_ARN + "#db.password", + SecretReference.parse(DB_SECRET_ARN + "#db.password").toString()); + } + + @Test + public void testNullContentRejected() { + assertThrows(NullPointerException.class, () -> SecretReference.parse(null)); + } + + @Test + public void testBlankContentRejected() { + assertThrows(IllegalArgumentException.class, () -> SecretReference.parse(" ")); + } + + @Test + public void testNonArnRejected() { + assertThrows( + IllegalArgumentException.class, () -> SecretReference.parse("just-a-plain-secret-name")); + } + + @Test + public void testWrongServiceRejected() { + // Correct ARN shape but not a secretsmanager resource. + assertThrows( + IllegalArgumentException.class, + () -> SecretReference.parse("arn:aws:s3:ap-northeast-1:123456789012:secret:foo")); + } + + @Test + public void testWrongResourceTypeRejected() { + assertThrows( + IllegalArgumentException.class, + () -> SecretReference.parse("arn:aws:secretsmanager:ap-northeast-1:123456789012:key:foo")); + } + + @Test + public void testMissingRegionRejected() { + assertThrows( + IllegalArgumentException.class, + () -> SecretReference.parse("arn:aws:secretsmanager::123456789012:secret:foo")); + } + + @Test + public void testMissingNameRejected() { + assertThrows( + IllegalArgumentException.class, + () -> SecretReference.parse("arn:aws:secretsmanager:ap-northeast-1:123456789012:secret:")); + } +} diff --git a/dev/deps/dependencies-hadoop-2-spark-3.3 b/dev/deps/dependencies-hadoop-2-spark-3.3 index d1fa96bf9d..9703bcca08 100644 --- a/dev/deps/dependencies-hadoop-2-spark-3.3 +++ b/dev/deps/dependencies-hadoop-2-spark-3.3 @@ -382,6 +382,7 @@ scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar scala-reflect/2.12.15//scala-reflect-2.12.15.jar scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar sdk-core/2.24.12//sdk-core-2.24.12.jar +secretsmanager/2.24.12//secretsmanager-2.24.12.jar simpleclient/0.16.0//simpleclient-0.16.0.jar simpleclient_common/0.16.0//simpleclient_common-0.16.0.jar simpleclient_httpserver/0.16.0//simpleclient_httpserver-0.16.0.jar diff --git a/dev/deps/dependencies-hadoop-3-spark-3.5 b/dev/deps/dependencies-hadoop-3-spark-3.5 index c5f4fe97d6..4f45caf2e9 100644 --- a/dev/deps/dependencies-hadoop-3-spark-3.5 +++ b/dev/deps/dependencies-hadoop-3-spark-3.5 @@ -349,6 +349,7 @@ scala-parser-combinators_2.12/2.3.0//scala-parser-combinators_2.12-2.3.0.jar scala-reflect/2.12.18//scala-reflect-2.12.18.jar scala-xml_2.12/2.1.0//scala-xml_2.12-2.1.0.jar sdk-core/2.24.12//sdk-core-2.24.12.jar +secretsmanager/2.24.12//secretsmanager-2.24.12.jar simpleclient/0.16.0//simpleclient-0.16.0.jar simpleclient_common/0.16.0//simpleclient_common-0.16.0.jar simpleclient_httpserver/0.16.0//simpleclient_httpserver-0.16.0.jar diff --git a/pom.xml b/pom.xml index a0430402c8..e8e766b325 100644 --- a/pom.xml +++ b/pom.xml @@ -741,6 +741,12 @@ ${awssdk.version} + + software.amazon.awssdk + secretsmanager + ${awssdk.version} + + software.amazon.awssdk url-connection-client