diff --git a/evcache-core/src/main/java/com/netflix/evcache/EVCacheImpl.java b/evcache-core/src/main/java/com/netflix/evcache/EVCacheImpl.java index 1a3c2588..40711158 100644 --- a/evcache-core/src/main/java/com/netflix/evcache/EVCacheImpl.java +++ b/evcache-core/src/main/java/com/netflix/evcache/EVCacheImpl.java @@ -6,6 +6,7 @@ import com.netflix.archaius.api.PropertyRepository; import com.netflix.evcache.EVCacheInMemoryCache.DataNotFoundException; import com.netflix.evcache.EVCacheLatch.Policy; +import com.netflix.evcache.config.EVCacheTranscoderProperties; import com.netflix.evcache.dto.KeyMapDto; import com.netflix.evcache.event.EVCacheEvent; import com.netflix.evcache.event.EVCacheEventListener; @@ -76,6 +77,7 @@ public class EVCacheImpl implements EVCache, EVCacheImplMBean { private static final Logger log = LoggerFactory.getLogger(EVCacheImpl.class); + private static final int COMPRESSION_THRESHOLD_BYTES = Integer.MAX_VALUE; private final Clock clock; private final String _appName; @@ -164,9 +166,9 @@ public class EVCacheImpl implements EVCache, EVCacheImplMBean { this.maxHashLength = propertyRepository.get(appName + ".max.hash.length", Integer.class).orElse(-1); this.encoderBase = propertyRepository.get(appName + ".hash.encoder", String.class).orElse("base64"); this.autoHashKeys = propertyRepository.get(_appName + ".auto.hash.keys", Boolean.class).orElseGet("evcache.auto.hash.keys").orElse(false); - this.evcacheValueTranscoder = new EVCacheTranscoder(_appName, propertyRepository); - evcacheValueTranscoder.setCompressionThreshold(Integer.MAX_VALUE); - + final EVCacheTranscoderProperties evCacheTranscoderProperties = new EVCacheTranscoderProperties(_appName, propertyRepository); + this.evcacheValueTranscoder = new EVCacheTranscoder(evCacheTranscoderProperties); + evcacheValueTranscoder.setCompressionThreshold(COMPRESSION_THRESHOLD_BYTES); // default max key length is 200, instead of using what is defined in MemcachedClientIF.MAX_KEY_LENGTH (250). This is to accommodate // auto key prepend with appname for duet feature. this.maxKeyLength = propertyRepository.get(_appName + ".max.key.length", Integer.class).orElseGet("evcache.max.key.length").orElse(200); diff --git a/evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java b/evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java index 1e73c384..77eabbf8 100644 --- a/evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java +++ b/evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java @@ -24,8 +24,10 @@ import com.github.luben.zstd.Zstd; import com.github.luben.zstd.ZstdInputStream; -import com.netflix.archaius.api.Property; +import com.netflix.evcache.config.EVCacheTranscoderProperties; +import com.netflix.evcache.config.EVCacheTranscoderProperties.CompressionAlgorithm; import com.netflix.evcache.metrics.EVCacheMetricsFactory; +import com.netflix.evcache.util.EVCacheConfig; import com.netflix.spectator.api.BasicTag; import com.netflix.spectator.api.DistributionSummary; import com.netflix.spectator.api.Tag; @@ -70,16 +72,12 @@ public class EVCacheSerializingTranscoder extends BaseSerializingTranscoder impl static final int SPECIAL_DOUBLE = (7 << 8); static final int SPECIAL_BYTEARRAY = (8 << 8); - public enum CompressionAlgorithm { GZIP, ZSTD } - - public static final int DEFAULT_ZSTD_COMPRESSION_LEVEL = 3; - private static final int ZSTD_MAGIC = 0xFD2FB528; private final TranscoderUtils tu = new TranscoderUtils(true); - private Property compressionAlgorithmProperty; - private Property zstdLevelProperty; - protected final String appName; + + protected final EVCacheTranscoderProperties properties; + private final EnumMap compressionRatioSummaries; /** @@ -90,19 +88,26 @@ public EVCacheSerializingTranscoder() { } /** - * Get a serializing transcoder that specifies the max data size. + * Get a serializing transcoder that specifies the max data size. Builds a default + * {@link EVCacheTranscoderProperties} bundle from {@link EVCacheConfig#getInstance()} — + * subclasses/callers that want per-app resolution should use + * {@link #EVCacheSerializingTranscoder(EVCacheTranscoderProperties, int)}. */ public EVCacheSerializingTranscoder(int max) { - this(null, max); + this(new EVCacheTranscoderProperties(null, EVCacheConfig.getInstance().getPropertyRepository()), max); } /** - * Get a serializing transcoder that specifies the owning app name and the max data size. + * Get a serializing transcoder with the supplied transcoder-property bundle. The bundle is + * exposed to subclasses via {@link #properties} so downstream transcoders can consult the + * same three-level (per-app → global → static default) resolution chain. Compression + * algorithm and zstd level are resolved dynamically on every {@code compress()} call via + * the {@link Property} handles below, so live FP updates propagate without JVM restart. */ - public EVCacheSerializingTranscoder(String appName, int max) { + public EVCacheSerializingTranscoder(EVCacheTranscoderProperties properties, int max) { super(max); - this.appName = appName; - this.compressionRatioSummaries = buildCompressionRatioSummaries(appName); + this.properties = properties; + this.compressionRatioSummaries = buildCompressionRatioSummaries(properties.getAppName()); } private static EnumMap buildCompressionRatioSummaries(String appName) { @@ -118,14 +123,6 @@ private static EnumMap buildCompressi return summaries; } - public void setCompressionAlgorithmProperty(Property algorithmProperty) { - this.compressionAlgorithmProperty = algorithmProperty; - } - - public void setCompressionLevelProperty(Property levelProperty) { - this.zstdLevelProperty = levelProperty; - } - @Override public boolean asyncDecode(CachedData d) { if ((d.getFlags() & COMPRESSED) != 0 || (d.getFlags() & SERIALIZED) != 0) { @@ -242,19 +239,17 @@ public CachedData encode(Object o) { protected byte[] compress(byte[] in) { if (in == null) throw new NullPointerException("Can't compress null"); - CompressionAlgorithm compressionAlgorithm = compressionAlgorithmProperty == null ? CompressionAlgorithm.GZIP - : CompressionAlgorithm.valueOf(compressionAlgorithmProperty.orElse(CompressionAlgorithm.GZIP.name()).get().toUpperCase()); + CompressionAlgorithm compressionAlgorithm = properties.getCompressionAlgorithmProperty().get(); byte[] compressed; switch (compressionAlgorithm) { case ZSTD: - int zstdLevel = zstdLevelProperty == null ? DEFAULT_ZSTD_COMPRESSION_LEVEL - : zstdLevelProperty.orElse(DEFAULT_ZSTD_COMPRESSION_LEVEL).get(); - logger.debug("algorithm: {}, level: {}, appName: {}", compressionAlgorithm, zstdLevel, appName); + int zstdLevel = properties.getZstdCompressionLevelProperty().get(); + logger.debug("algorithm: {}, level: {}, appName: {}", compressionAlgorithm, zstdLevel, properties.getAppName()); compressed = Zstd.compress(in, zstdLevel); break; case GZIP: - logger.debug("algorithm: {}, appName: {}", compressionAlgorithm, appName); + logger.debug("algorithm: {}, appName: {}", compressionAlgorithm, properties.getAppName()); compressed = super.compress(in); break; default: @@ -293,7 +288,8 @@ private byte[] decompressZstd(byte[] in) { } // Slow path: declared size is 0, unknown (-1), or invalid (-2) — stream-decode and let // ZstdInputStream surface any frame errors. - logger.warn("Zstd frame missing content-size header (getFrameContentSize={}); falling back to stream decode. appName={}", originalSize, appName); + logger.warn("Zstd frame missing content-size header (getFrameContentSize={}); falling back to stream decode. appName={}", + originalSize, properties.getAppName()); ZstdInputStream zis = null; try { zis = new ZstdInputStream(new ByteArrayInputStream(in)); diff --git a/evcache-core/src/main/java/com/netflix/evcache/EVCacheTranscoder.java b/evcache-core/src/main/java/com/netflix/evcache/EVCacheTranscoder.java index af267e3e..d540460d 100644 --- a/evcache-core/src/main/java/com/netflix/evcache/EVCacheTranscoder.java +++ b/evcache-core/src/main/java/com/netflix/evcache/EVCacheTranscoder.java @@ -1,75 +1,76 @@ package com.netflix.evcache; -import com.netflix.archaius.api.Property; -import com.netflix.archaius.api.PropertyRepository; +import com.netflix.evcache.config.EVCacheTranscoderProperties; +import com.netflix.evcache.pool.EVCacheValue; +import com.netflix.evcache.pool.EVCacheValueSerde; import com.netflix.evcache.util.EVCacheConfig; import net.spy.memcached.CachedData; +import static com.netflix.evcache.config.EVCacheTranscoderProperties.DEFAULT_COMPRESSION_THRESHOLD_BYTES; +import static com.netflix.evcache.config.EVCacheTranscoderProperties.DEFAULT_MAX_DATA_SIZE_BYTES; +import static com.netflix.evcache.config.EVCacheTranscoderProperties.Key.COMPRESSION_THRESHOLD_BYTES; +import static com.netflix.evcache.config.EVCacheTranscoderProperties.Key.MAX_DATA_SIZE_BYTES; + public class EVCacheTranscoder extends EVCacheSerializingTranscoder { - public EVCacheTranscoder() { - this((String) null); + /** + * @param properties the transcoder property bundle. + * {@link EVCacheTranscoderProperties.Key#MAX_DATA_SIZE_BYTES} and + * {@link EVCacheTranscoderProperties.Key#COMPRESSION_THRESHOLD_BYTES} + * are read from the bundle at construction and set as fields on the + * underlying {@link EVCacheSerializingTranscoder} for legacy reasons; + * new properties should be added to {@link EVCacheTranscoderProperties} + * rather than plumbed through further constructor arguments. + */ + public EVCacheTranscoder(EVCacheTranscoderProperties properties) { + this(properties.getProperty(MAX_DATA_SIZE_BYTES, Integer.class, DEFAULT_MAX_DATA_SIZE_BYTES), + properties.getProperty(COMPRESSION_THRESHOLD_BYTES, Integer.class, DEFAULT_COMPRESSION_THRESHOLD_BYTES), + properties + ); } - public EVCacheTranscoder(String appName) { - this(appName, EVCacheConfig.getInstance().getPropertyRepository()); + public EVCacheTranscoder() { + this(new EVCacheTranscoderProperties(null, EVCacheConfig.getInstance().getPropertyRepository())); } public EVCacheTranscoder(int max) { - this(null, max); - } - - public EVCacheTranscoder(String appName, int max) { - this(appName, EVCacheConfig.getInstance().getPropertyRepository(), max); + this(max, new EVCacheTranscoderProperties(null, EVCacheConfig.getInstance().getPropertyRepository())); } public EVCacheTranscoder(int max, int compressionThreshold) { - this(null, max, compressionThreshold); - } - - public EVCacheTranscoder(String appName, int max, int compressionThreshold) { - this(appName, EVCacheConfig.getInstance().getPropertyRepository(), max, compressionThreshold); + this(max, compressionThreshold, new EVCacheTranscoderProperties(null, EVCacheConfig.getInstance().getPropertyRepository())); } - /** - * Repository-aware constructors. The compression algorithm/level are read dynamically from the - * supplied {@link PropertyRepository}, so callers must pass the repository that is wired to Fast - * Properties (e.g. {@code poolManager.getEVCacheConfig().getPropertyRepository()}) for FP overrides - * to take effect. The no-repository constructors above fall back to {@link EVCacheConfig#getInstance()}. - */ - public EVCacheTranscoder(String appName, PropertyRepository config) { - this(appName, config, config.get("default.evcache.max.data.size", Integer.class).orElse(20 * 1024 * 1024).get()); + private EVCacheTranscoder(int max, EVCacheTranscoderProperties properties) { + this(max, properties.getProperty(COMPRESSION_THRESHOLD_BYTES, Integer.class, DEFAULT_COMPRESSION_THRESHOLD_BYTES), properties); } - public EVCacheTranscoder(String appName, PropertyRepository config, int max) { - this(appName, config, max, config.get("default.evcache.compression.threshold", Integer.class).orElse(120).get()); + private EVCacheTranscoder(int max, int compressionThreshold, EVCacheTranscoderProperties properties) { + super(properties, max); + setCompressionThreshold(compressionThreshold); } - public EVCacheTranscoder(String appName, PropertyRepository config, int max, int compressionThreshold) { - super(appName, max); - setCompressionThreshold(compressionThreshold); - Property algoProperty = getProperty(config, "default.evcacheclient.compression.algo", String.class); - setCompressionAlgorithmProperty(algoProperty); - Property zstdLevelProperty = getProperty(config, "default.evcacheclient.compression.zstd.level", Integer.class); - setCompressionLevelProperty(zstdLevelProperty); + @Override + public CachedData encode(Object o) { + if (o != null && o instanceof CachedData) return (CachedData) o; + return super.encode(o); } - /** - * Resolves a property preferring the appName-prefixed key and falling back to the global {@code evcache.*} key when - * no app-specific override exists. - */ - private Property getProperty(PropertyRepository config, String key, Class type) { - if (appName == null || appName.isEmpty()) { - return config.get("default." + key, type); + @Override + protected byte[] serialize(Object o) { + if (this.properties.isBinarySerializationEnabled() && o instanceof EVCacheValue) { + return EVCacheValueSerde.serialize((EVCacheValue) o); } - return config.get(appName + "." + key, type).orElseGet(key); + return super.serialize(o); } @Override - public CachedData encode(Object o) { - if (o != null && o instanceof CachedData) return (CachedData) o; - return super.encode(o); + protected Object deserialize(byte[] in) { + if (EVCacheValueSerde.isBinaryFormat(in)) { + return EVCacheValueSerde.deserialize(in); + } + return super.deserialize(in); } } diff --git a/evcache-core/src/main/java/com/netflix/evcache/config/EVCacheTranscoderProperties.java b/evcache-core/src/main/java/com/netflix/evcache/config/EVCacheTranscoderProperties.java new file mode 100644 index 00000000..f2cd11ce --- /dev/null +++ b/evcache-core/src/main/java/com/netflix/evcache/config/EVCacheTranscoderProperties.java @@ -0,0 +1,135 @@ +package com.netflix.evcache.config; + +import com.netflix.archaius.api.Property; +import com.netflix.archaius.api.PropertyRepository; + +import static com.netflix.evcache.config.EVCacheTranscoderProperties.Key.BINARY_SERIALIZATION_ENABLED; + +/** + * Properties related to {@link com.netflix.evcache.EVCacheTranscoder} + * behavior. + * + *

Every property resolves at construction through: + * + *

    + *
  1. Per-app override: {@code .}
    2. + *
  2. Global default: {@code }
    3. + *
  3. Static default: default value supplied as an argument
    4. + *
+ * + *

+ * Static properties should be cached as a field for fast access. + * Dynamic properties get be accessed {@link #getProperty(Key, Class, Object)} + * + */ +public final class EVCacheTranscoderProperties { + + public static final boolean DEFAULT_BINARY_SERIALIZATION_ENABLED = false; + public static final int DEFAULT_MAX_DATA_SIZE_BYTES = 20 * 1024 * 1024; + public static final int DEFAULT_COMPRESSION_THRESHOLD_BYTES = 120; + public static final CompressionAlgorithm DEFAULT_COMPRESSION_ALGORITHM = CompressionAlgorithm.GZIP; + public static final int DEFAULT_COMPRESSION_ZSTD_LEVEL = 3; + + public enum CompressionAlgorithm { GZIP, ZSTD } + + public enum Key { + BINARY_SERIALIZATION_ENABLED("binary.serialization.enabled", "default.evcache.binary.serialization.enabled"), + MAX_DATA_SIZE_BYTES("max.data.size", "default.evcache.max.data.size"), + COMPRESSION_THRESHOLD_BYTES("compression.threshold", "default.evcache.compression.threshold"), + COMPRESSION_ALGORITHM("compression.algorithm", "default.evcache.compression.algorithm"), + COMPRESSION_ZSTD_LEVEL("compression.zstd.level", "default.evcache.compression.zstd.level"); + + final String appKeySuffix; + final String globalKey; + + Key(String appKeySuffix, String globalKey) { + this.appKeySuffix = appKeySuffix; + this.globalKey = globalKey; + } + } + + private final String appName; + private final PropertyRepository propertyRepository; + + private final boolean binarySerializationEnabled; + private final Property compressionAlgorithmProperty; + private final Property zstdCompressionLevelProperty; + + /** + * Construct the bundle and snapshot every property via the three-level resolution chain. + * + * @param appName the EVCache app name used as the per-app override prefix + * (e.g. {@code "EVCACHE_FOO"}). When {@code null} or empty the + * per-app step is skipped and resolution starts at the global + * key — useful for the no-app transcoder constructors and for + * callers that only want fleet-wide defaults. + * @param propertyRepository the Archaius2 PropertyRepository to resolve against. Never null; + * pass {@code EVCacheConfig.getInstance().getPropertyRepository()} + * for the production wiring. + */ + public EVCacheTranscoderProperties(String appName, PropertyRepository propertyRepository) { + this.appName = appName; + this.propertyRepository = propertyRepository; + this.binarySerializationEnabled = getProperty(appName, propertyRepository, + BINARY_SERIALIZATION_ENABLED, Boolean.class, DEFAULT_BINARY_SERIALIZATION_ENABLED).get(); + this.compressionAlgorithmProperty = getProperty(appName, propertyRepository, + Key.COMPRESSION_ALGORITHM, String.class, DEFAULT_COMPRESSION_ALGORITHM.name()) + .map(s -> CompressionAlgorithm.valueOf(s.toUpperCase())); + this.zstdCompressionLevelProperty = getProperty(appName, propertyRepository, + Key.COMPRESSION_ZSTD_LEVEL, Integer.class, DEFAULT_COMPRESSION_ZSTD_LEVEL); + } + + public boolean isBinarySerializationEnabled() { + return binarySerializationEnabled; + } + + public String getAppName() { + return appName; + } + + /** + * Read a property dynamically (re-evaluates on every call), with the same per-app -> global -> + * static-default chain used for the cached fields above. + */ + public T getProperty(Key key, Class type, T defaultValue) { + return getProperty(appName, propertyRepository, key, type, defaultValue).get(); + } + + /** + * Resolve the Archaius {@link Property} handle for the given key. Callers can hold the handle + * as a final field and invoke {@link Property#get()} to read the current value; each call + * observes live FP updates without re-resolving the handle. + */ + public Property resolveProperty(Key key, Class type, T defaultValue) { + return getProperty(appName, propertyRepository, key, type, defaultValue); + } + + /** + * Live-updating {@link Property} handle for the transcoder compression algorithm. Calling + * {@code .get()} returns the current {@link CompressionAlgorithm} value; underlying storage + * is a String property (case-insensitive) so ops can set the FP as {@code "gzip"} or + * {@code "ZSTD"} interchangeably. Handle is resolved once at bundle construction and + * shared across callers — {@code .get()} on it always observes the latest FP value. + */ + public Property getCompressionAlgorithmProperty() { + return compressionAlgorithmProperty; + } + + /** + * Live-updating {@link Property} handle for the zstd compression level. Resolved once at + * bundle construction. + */ + public Property getZstdCompressionLevelProperty() { + return zstdCompressionLevelProperty; + } + + private static Property getProperty(String appName, PropertyRepository propertyRepository, + Key key, Class type, T defaultValue) { + if (appName == null || appName.isEmpty()) { + return propertyRepository.get(key.globalKey, type).orElse(defaultValue); + } + return propertyRepository.get(appName + "." + key.appKeySuffix, type) + .orElseGet(key.globalKey) + .orElse(defaultValue); + } +} diff --git a/evcache-core/src/main/java/com/netflix/evcache/connection/BaseAsciiConnectionFactory.java b/evcache-core/src/main/java/com/netflix/evcache/connection/BaseAsciiConnectionFactory.java index 2b12c3b8..d90aa490 100644 --- a/evcache-core/src/main/java/com/netflix/evcache/connection/BaseAsciiConnectionFactory.java +++ b/evcache-core/src/main/java/com/netflix/evcache/connection/BaseAsciiConnectionFactory.java @@ -115,8 +115,8 @@ public BlockingQueue createWriteOperationQueue() { } public Transcoder getDefaultTranscoder() { - return new EVCacheTranscoder(appName, - client.getPool().getEVCacheClientPoolManager().getEVCacheConfig().getPropertyRepository()); + return new EVCacheTranscoder(new com.netflix.evcache.config.EVCacheTranscoderProperties(appName, + client.getPool().getEVCacheClientPoolManager().getEVCacheConfig().getPropertyRepository())); } public FailureMode getFailureMode() { diff --git a/evcache-core/src/main/java/com/netflix/evcache/connection/BaseConnectionFactory.java b/evcache-core/src/main/java/com/netflix/evcache/connection/BaseConnectionFactory.java index 37d300f9..22fdf7b2 100644 --- a/evcache-core/src/main/java/com/netflix/evcache/connection/BaseConnectionFactory.java +++ b/evcache-core/src/main/java/com/netflix/evcache/connection/BaseConnectionFactory.java @@ -109,8 +109,8 @@ public BlockingQueue createWriteOperationQueue() { } public Transcoder getDefaultTranscoder() { - return new EVCacheTranscoder(appName, - client.getPool().getEVCacheClientPoolManager().getEVCacheConfig().getPropertyRepository()); + return new EVCacheTranscoder(new com.netflix.evcache.config.EVCacheTranscoderProperties(appName, + client.getPool().getEVCacheClientPoolManager().getEVCacheConfig().getPropertyRepository())); } public FailureMode getFailureMode() { diff --git a/evcache-core/src/main/java/com/netflix/evcache/pool/EVCacheValueSerde.java b/evcache-core/src/main/java/com/netflix/evcache/pool/EVCacheValueSerde.java new file mode 100644 index 00000000..d65b4078 --- /dev/null +++ b/evcache-core/src/main/java/com/netflix/evcache/pool/EVCacheValueSerde.java @@ -0,0 +1,196 @@ +package com.netflix.evcache.pool; + +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import org.apache.commons.codec.binary.Hex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Length-prefixed binary wire format for the {@link EVCacheValue} envelope. EVCache wraps a + * value in an {@code EVCacheValue} when the canonical key has to be hashed (see + * {@code EVCacheImpl.getEVCacheKey}) so the pre-hash key is preserved for collision detection. + * + * 
+ * [byte 0: magic 0x0C][byte 1: reserved/version 0x00]
+ * [int keyLen][key UTF-8 bytes]
+ * [int valLen][value bytes]
+ * [int flags][long ttl][long createTime]
+ * [... optional extension fields appended by newer writers ...]
+ *
+ * + *
    + *
  • Magic {@code 0x0C} disambiguates from Java {@code ObjectOutputStream} (starts + * {@code 0xAC 0xED}); callers route via {@link #isBinaryFormat(byte[])}.
    • + *
  • Reserved/version byte is currently {@code 0x00}, read-and-ignored. Bump only + * for breaking changes (see Upgrades).
    • + *
  • End of envelope is implicit at {@code bytes.length}. There is no declared body + * length on the wire; bytes past the last known field are treated as extension data for + * additive forward-compat (see Upgrades).
    • + *
  • Byte order: big-endian / network, set explicitly on both sides.
    • + *
  • Error contract: any corrupt/truncated input returns {@code null} after a WARN + * log identifying the failing field and a (truncated) hex dump of the bytes. Matches + * {@code BaseSerializingTranscoder}'s resilience contract — caller sees a cache miss.
    • + *
+ * + *

Upgrades

+ * + *

Additive optional (non-breaking). Append a new field at the end of the envelope, + * after {@code createTime}. Older readers stop after the known fields and never look at the + * extension bytes. Newer readers MUST gate each added field with {@code buffer.hasRemaining()} + * and supply a default when absent — they will encounter items written by old writers + * (in cache until TTL expires) that don't contain the field. Only works when a graceful + * default exists. A new required field has no acceptable default and is therefore + * Breaking, not additive. + * + *

Breaking (field reorder, type widen, semantic change, new required field): + * rollout MUST be reader-before-writer — items written by an early writer would be + * silently misparsed by lagging readers and survive until TTL. + *

    + *
  1. Ship a version-aware reader that branches on byte 1: {@code 0x00} stays on the current + * decoder, the new value routes to the new decoder, unknown values + * {@link #logCorruption(byte[], String)} and return {@code null}. Deploy to 100% of every + * consumer that calls {@link #deserialize} (clients, admin tools, cache warmers, + * replicators).
    2. + *
  2. Wait for the longer of (full reader rollout) and (max item TTL).
    3. + *
  3. Then ship the new writer gated by a per-app FastProperty so canary is possible.
    4. + *
  4. Never reuse a version byte value for a different layout.
    5. + *
  5. Keep the old decoder path indefinitely — items live until their TTL expires.
    6. + *
+ */ +public final class EVCacheValueSerde { + + private static final Logger log = LoggerFactory.getLogger(EVCacheValueSerde.class); + + static final byte BINARY_SERDE_MAGIC_CONSTANT_BYTE = 0x0C; // 12 + private static final byte RESERVED_VERSION_BYTE = 0x00; + + private static final int CORRUPT_PAYLOAD_LOG_LIMIT = 1024; + + private EVCacheValueSerde() { + // Utility class; not instantiable. + } + + /** True iff {@code bytes} starts with the binary envelope magic byte. */ + public static boolean isBinaryFormat(byte[] bytes) { + return bytes != null && bytes.length > 0 && bytes[0] == BINARY_SERDE_MAGIC_CONSTANT_BYTE; + } + + /** + * Encode an {@link EVCacheValue} into its compact binary envelope. Key and value must be + * non-null — the {@link com.netflix.evcache.EVCacheTranscoder} / {@code CachedData} pipeline + * above already rejects nulls. + */ + public static byte[] serialize(EVCacheValue v) { + final byte[] keyBytes = v.getKey().getBytes(StandardCharsets.UTF_8); + final byte[] valueBytes = v.getValue(); + + final int bufferSize = + Byte.BYTES + Byte.BYTES // magic + reserved/version + + Integer.BYTES + keyBytes.length // keyLen + key + + Integer.BYTES + valueBytes.length // valLen + value + + Integer.BYTES // flags + + Long.BYTES // ttl + + Long.BYTES; // createTime + final ByteBuffer buffer = ByteBuffer.allocate(bufferSize).order(ByteOrder.BIG_ENDIAN); + + buffer.put(BINARY_SERDE_MAGIC_CONSTANT_BYTE); + buffer.put(RESERVED_VERSION_BYTE); + + buffer.putInt(keyBytes.length); + buffer.put(keyBytes); + buffer.putInt(valueBytes.length); + buffer.put(valueBytes); + buffer.putInt(v.getFlags()); + buffer.putLong(v.getTTL()); + buffer.putLong(v.getCreateTimeUTC()); + + return buffer.array(); + } + + /** + * Decode the binary envelope. Length prefixes are bounds-checked before allocation. A + * truncated or malformed payload returns {@code null} after a WARN log identifying the + * failing field. Bytes remaining past the known fields are not read — they're reserved for + * additive extension fields appended by newer writers (see Upgrades). + */ + public static EVCacheValue deserialize(byte[] bytes) { + String field = "magic"; + try { + final ByteBuffer buffer = ByteBuffer.wrap(bytes).order(ByteOrder.BIG_ENDIAN); + + final byte magic = buffer.get(); + if (BINARY_SERDE_MAGIC_CONSTANT_BYTE != magic) { + logCorruption(bytes, "Invalid magic constant: " + magic); + return null; + } + field = "reserved"; + buffer.get(); + + field = "keyLength"; + final int keyLength = buffer.getInt(); + if (keyLength < 0 || keyLength > buffer.remaining()) { + logCorruption(bytes, "Invalid keyLength: " + keyLength + ", remaining=" + buffer.remaining()); + return null; + } + field = "key"; + final byte[] keyBytes = new byte[keyLength]; + buffer.get(keyBytes); + final String key = new String(keyBytes, StandardCharsets.UTF_8); + + field = "valueLength"; + final int valueLength = buffer.getInt(); + if (valueLength < 0 || valueLength > buffer.remaining()) { + logCorruption(bytes, "Invalid valueLength: " + valueLength + ", remaining=" + buffer.remaining()); + return null; + } + field = "value"; + final byte[] valueBytes = new byte[valueLength]; + buffer.get(valueBytes); + + field = "flags"; + final int flags = buffer.getInt(); + field = "ttl"; + final long ttl = buffer.getLong(); + field = "createTime"; + final long createTime = buffer.getLong(); + + // Any remaining bytes are forward-compat extension fields a newer writer appended; + // an older reader (this one) leaves them unread. + + return new EVCacheValue(key, valueBytes, flags, ttl, createTime); + } catch (BufferUnderflowException e) { + logCorruption(bytes, "BufferUnderflow at field '" + field + "'"); + return null; + } catch (Exception e) { + log.warn("Uncaught exception decoding {} bytes of EVCacheValue binary envelope at field '{}'", + bytes.length, field, e); + return null; + } + } + + /** + * Warn-log a corruption event with byte length, failure reason, and a (truncated) hex dump. + * No Throwable — corruption is expected/recoverable at WARN level; a stack trace would be + * noise. Hex capped at {@value #CORRUPT_PAYLOAD_LOG_LIMIT} bytes. + */ + private static void logCorruption(byte[] bytes, String error) { + log.warn("Failed to deserialize {} bytes of EVCacheValue binary envelope, error={}, payload hex: {}", + bytes.length, error, toHex(bytes, CORRUPT_PAYLOAD_LOG_LIMIT)); + } + + private static String toHex(byte[] bytes, int maxBytes) { + if (bytes == null) { + return "null"; + } + if (bytes.length <= maxBytes) { + return Hex.encodeHexString(bytes); + } + return Hex.encodeHexString(Arrays.copyOf(bytes, maxBytes)) + + "...(truncated, total=" + bytes.length + " bytes)"; + } +} diff --git a/evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java b/evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java index 576d2bc6..02ab74ee 100644 --- a/evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java +++ b/evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java @@ -3,8 +3,8 @@ import com.netflix.archaius.DefaultPropertyFactory; import com.netflix.archaius.api.PropertyRepository; import com.netflix.archaius.config.DefaultSettableConfig; +import com.netflix.evcache.config.EVCacheTranscoderProperties; import com.netflix.evcache.metrics.EVCacheMetricsFactory; -import com.netflix.evcache.util.EVCacheConfig; import com.netflix.spectator.api.DefaultRegistry; import com.netflix.spectator.api.Id; import com.netflix.spectator.api.Meter; @@ -18,28 +18,45 @@ public class EVCacheSerializingTranscoderTest { + private static final String GLOBAL_ALGO_KEY = "default.evcache.compression.algorithm"; + private static final String GLOBAL_ZSTD_LEVEL_KEY = "default.evcache.compression.zstd.level"; + private static final String PER_APP_ALGO_SUFFIX = ".compression.algorithm"; + private static final String PER_APP_ZSTD_LEVEL_SUFFIX = ".compression.zstd.level"; + private EVCacheSerializingTranscoder buildTranscoder(String algo, Integer level) { - DefaultSettableConfig config = new DefaultSettableConfig(); - config.setProperty("test.algo", algo); - if (level != null) config.setProperty("test.level", level); - PropertyRepository repo = new DefaultPropertyFactory(config); - EVCacheSerializingTranscoder t = new EVCacheSerializingTranscoder(CachedData.MAX_SIZE); - t.setCompressionAlgorithmProperty(repo.get("test.algo", String.class)); - t.setCompressionLevelProperty(repo.get("test.level", Integer.class)); + return buildTranscoder(null, algo, level); + } + + private EVCacheSerializingTranscoder buildTranscoder(String appName, String algo, Integer level) { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + if (algo != null) cfg.setProperty(GLOBAL_ALGO_KEY, algo); + if (level != null) cfg.setProperty(GLOBAL_ZSTD_LEVEL_KEY, level); + PropertyRepository repo = new DefaultPropertyFactory(cfg); + return new EVCacheSerializingTranscoder( + new EVCacheTranscoderProperties(appName, repo), CachedData.MAX_SIZE); + } + + private EVCacheTranscoder buildEVCacheTranscoder(String appName, String algo, Integer level, int compressionThreshold) { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + if (algo != null) cfg.setProperty(GLOBAL_ALGO_KEY, algo); + if (level != null) cfg.setProperty(GLOBAL_ZSTD_LEVEL_KEY, level); + PropertyRepository repo = new DefaultPropertyFactory(cfg); + EVCacheTranscoder t = new EVCacheTranscoder(new EVCacheTranscoderProperties(appName, repo)); + t.setCompressionThreshold(compressionThreshold); return t; } @Test public void testEnumValues() { - assertEquals(EVCacheSerializingTranscoder.CompressionAlgorithm.valueOf("GZIP"), - EVCacheSerializingTranscoder.CompressionAlgorithm.GZIP); - assertEquals(EVCacheSerializingTranscoder.CompressionAlgorithm.valueOf("ZSTD"), - EVCacheSerializingTranscoder.CompressionAlgorithm.ZSTD); + assertEquals(EVCacheTranscoderProperties.CompressionAlgorithm.valueOf("GZIP"), + EVCacheTranscoderProperties.CompressionAlgorithm.GZIP); + assertEquals(EVCacheTranscoderProperties.CompressionAlgorithm.valueOf("ZSTD"), + EVCacheTranscoderProperties.CompressionAlgorithm.ZSTD); } @Test public void testDefaultZstdLevelConstant() { - assertEquals(EVCacheSerializingTranscoder.DEFAULT_ZSTD_COMPRESSION_LEVEL, 3); + assertEquals(EVCacheTranscoderProperties.DEFAULT_COMPRESSION_ZSTD_LEVEL, 3); } @Test @@ -96,7 +113,6 @@ public void testZstdEncodeSetsZstdMagicBytes() { assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, "COMPRESSED flag must be set"); byte[] data = encoded.getData(); - // Zstd magic is 0xFD2FB528 in little-endian: bytes 0x28 0xB5 0x2F 0xFD assertEquals(data[0], (byte) 0x28, "Expected zstd magic byte 0"); assertEquals(data[1], (byte) 0xB5, "Expected zstd magic byte 1"); assertEquals(data[2], (byte) 0x2F, "Expected zstd magic byte 2"); @@ -125,7 +141,6 @@ public void testZstdRoundTrip() { @Test public void testGzipTranscoderDecodesZstdData() { - // zstd transcoder writes, gzip transcoder reads → cross-decode via magic-byte detection EVCacheSerializingTranscoder writer = buildTranscoder("ZSTD", null); writer.setCompressionThreshold(1); EVCacheSerializingTranscoder reader = buildTranscoder("GZIP", null); @@ -138,7 +153,6 @@ public void testGzipTranscoderDecodesZstdData() { @Test public void testZstdTranscoderDecodesGzipData() { - // gzip transcoder writes, zstd transcoder reads → cross-decode via magic-byte detection EVCacheSerializingTranscoder writer = buildTranscoder("GZIP", null); writer.setCompressionThreshold(1); EVCacheSerializingTranscoder reader = buildTranscoder("ZSTD", null); @@ -151,7 +165,8 @@ public void testZstdTranscoderDecodesGzipData() { @Test public void testEVCacheTranscoderDefaultsToGzip() { - EVCacheTranscoder transcoder = new EVCacheTranscoder((String) null, CachedData.MAX_SIZE, 0); + // No algo property set anywhere -> bundle default is GZIP. + EVCacheTranscoder transcoder = buildEVCacheTranscoder(null, null, null, 0); String original = "hello world hello world hello world hello world hello world"; CachedData encoded = transcoder.encode(original); assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, @@ -165,75 +180,51 @@ public void testEVCacheTranscoderDefaultsToGzip() { @Test public void testEVCacheTranscoderExplicitZstdAlgorithm() { - DefaultSettableConfig testConfig = new DefaultSettableConfig(); - testConfig.setProperty("evcacheclient.compression.algo", "ZSTD"); - testConfig.setProperty("evcacheclient.compression.zstd.level", - EVCacheSerializingTranscoder.DEFAULT_ZSTD_COMPRESSION_LEVEL); - PropertyRepository savedRepo = EVCacheConfig.getInstance().getPropertyRepository(); - EVCacheConfig.setPropertyRepository(new DefaultPropertyFactory(testConfig)); - try { - EVCacheTranscoder transcoder = new EVCacheTranscoder((String) null, CachedData.MAX_SIZE, 1); - String original = "hello world hello world hello world hello world hello world"; - CachedData encoded = transcoder.encode(original); - String decoded = (String) transcoder.decode(encoded); - assertEquals(decoded, original); - } finally { - EVCacheConfig.setPropertyRepository(savedRepo); - } + EVCacheTranscoder transcoder = buildEVCacheTranscoder(null, "ZSTD", + EVCacheTranscoderProperties.DEFAULT_COMPRESSION_ZSTD_LEVEL, 1); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = transcoder.encode(original); + String decoded = (String) transcoder.decode(encoded); + assertEquals(decoded, original); } @Test public void testAppNamePrefixedAlgoOverridesDefault() { - DefaultSettableConfig testConfig = new DefaultSettableConfig(); - testConfig.setProperty("evcacheclient.compression.algo", "GZIP"); - testConfig.setProperty("EVCACHE_TEST.evcacheclient.compression.algo", "ZSTD"); - PropertyRepository savedRepo = EVCacheConfig.getInstance().getPropertyRepository(); - EVCacheConfig.setPropertyRepository(new DefaultPropertyFactory(testConfig)); - try { - EVCacheTranscoder transcoder = new EVCacheTranscoder("EVCACHE_TEST", CachedData.MAX_SIZE, 1); - CachedData encoded = transcoder.encode("hello world hello world hello world hello world hello world"); - byte[] data = encoded.getData(); - assertEquals(data[0], (byte) 0x28, "app-specific ZSTD override must win over default GZIP"); - assertEquals(data[1], (byte) 0xB5, "app-specific ZSTD override must win over default GZIP"); - } finally { - EVCacheConfig.setPropertyRepository(savedRepo); - } + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(GLOBAL_ALGO_KEY, "GZIP"); + cfg.setProperty("EVCACHE_TEST" + PER_APP_ALGO_SUFFIX, "ZSTD"); + PropertyRepository repo = new DefaultPropertyFactory(cfg); + EVCacheTranscoder transcoder = new EVCacheTranscoder(new EVCacheTranscoderProperties("EVCACHE_TEST", repo)); + transcoder.setCompressionThreshold(1); + CachedData encoded = transcoder.encode("hello world hello world hello world hello world hello world"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x28, "app-specific ZSTD override must win over default GZIP"); + assertEquals(data[1], (byte) 0xB5, "app-specific ZSTD override must win over default GZIP"); } @Test public void testAppNameFallsBackToDefaultAlgoWhenNoOverride() { - DefaultSettableConfig testConfig = new DefaultSettableConfig(); - testConfig.setProperty("evcacheclient.compression.algo", "ZSTD"); - PropertyRepository savedRepo = EVCacheConfig.getInstance().getPropertyRepository(); - EVCacheConfig.setPropertyRepository(new DefaultPropertyFactory(testConfig)); - try { - EVCacheTranscoder transcoder = new EVCacheTranscoder("EVCACHE_NO_OVERRIDE", CachedData.MAX_SIZE, 1); - CachedData encoded = transcoder.encode("hello world hello world hello world hello world hello world"); - byte[] data = encoded.getData(); - assertEquals(data[0], (byte) 0x28, "must fall back to default ZSTD when no app-specific override exists"); - assertEquals(data[1], (byte) 0xB5, "must fall back to default ZSTD when no app-specific override exists"); - } finally { - EVCacheConfig.setPropertyRepository(savedRepo); - } + // Global says ZSTD, per-app for a *different* app; our transcoder falls back to global. + EVCacheTranscoder transcoder = buildEVCacheTranscoder("EVCACHE_NO_OVERRIDE", "ZSTD", null, 1); + CachedData encoded = transcoder.encode("hello world hello world hello world hello world hello world"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x28, "must fall back to global ZSTD when no app-specific override exists"); + assertEquals(data[1], (byte) 0xB5, "must fall back to global ZSTD when no app-specific override exists"); } @Test public void testAppNamePrefixedZstdLevelRoundTrip() { - DefaultSettableConfig testConfig = new DefaultSettableConfig(); - testConfig.setProperty("evcacheclient.compression.algo", "ZSTD"); - testConfig.setProperty("evcacheclient.compression.zstd.level", 1); - testConfig.setProperty("EVCACHE_TEST.evcacheclient.compression.zstd.level", 5); - PropertyRepository savedRepo = EVCacheConfig.getInstance().getPropertyRepository(); - EVCacheConfig.setPropertyRepository(new DefaultPropertyFactory(testConfig)); - try { - EVCacheTranscoder transcoder = new EVCacheTranscoder("EVCACHE_TEST", CachedData.MAX_SIZE, 1); - String original = "hello world hello world hello world hello world hello world"; - CachedData encoded = transcoder.encode(original); - String decoded = (String) transcoder.decode(encoded); - assertEquals(decoded, original, "app-specific zstd level override round-trip must succeed"); - } finally { - EVCacheConfig.setPropertyRepository(savedRepo); - } + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(GLOBAL_ALGO_KEY, "ZSTD"); + cfg.setProperty(GLOBAL_ZSTD_LEVEL_KEY, 1); + cfg.setProperty("EVCACHE_TEST" + PER_APP_ZSTD_LEVEL_SUFFIX, 5); + PropertyRepository repo = new DefaultPropertyFactory(cfg); + EVCacheTranscoder transcoder = new EVCacheTranscoder(new EVCacheTranscoderProperties("EVCACHE_TEST", repo)); + transcoder.setCompressionThreshold(1); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = transcoder.encode(original); + String decoded = (String) transcoder.decode(encoded); + assertEquals(decoded, original, "app-specific zstd level override round-trip must succeed"); } @Test @@ -242,12 +233,7 @@ public void testCompressionRatioMetricTaggedWithAppName() { Registry registry = new DefaultRegistry(); Spectator.globalRegistry().add(registry); try { - DefaultSettableConfig config = new DefaultSettableConfig(); - config.setProperty("test.algo", "GZIP"); - PropertyRepository repo = new DefaultPropertyFactory(config); - EVCacheSerializingTranscoder t = new EVCacheSerializingTranscoder(appName, CachedData.MAX_SIZE); - t.setCompressionAlgorithmProperty(repo.get("test.algo", String.class)); - t.setCompressionLevelProperty(repo.get("test.level", Integer.class)); + EVCacheSerializingTranscoder t = buildTranscoder(appName, "GZIP", null); t.setCompressionThreshold(0); t.encode("hello world hello world hello world hello world hello world"); @@ -297,62 +283,37 @@ private boolean hasCompressionRatioCacheTag(Registry registry, String appName) { @Test(expectedExceptions = IllegalArgumentException.class) public void testInvalidAlgorithmEnumThrows() { - EVCacheSerializingTranscoder.CompressionAlgorithm.valueOf("INVALID"); + EVCacheTranscoderProperties.CompressionAlgorithm.valueOf("INVALID"); } @Test public void testFPAlgorithmGzip() { - DefaultSettableConfig testConfig = new DefaultSettableConfig(); - testConfig.setProperty("evcacheclient.compression.algo", "GZIP"); - PropertyRepository savedRepo = EVCacheConfig.getInstance().getPropertyRepository(); - EVCacheConfig.setPropertyRepository(new DefaultPropertyFactory(testConfig)); - try { - EVCacheTranscoder transcoder = new EVCacheTranscoder((String) null, CachedData.MAX_SIZE, 1); - CachedData encoded = transcoder.encode("hello world hello world hello world hello world hello world"); - assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, - "COMPRESSED flag must be set"); - byte[] data = encoded.getData(); - assertEquals(data[0], (byte) 0x1f, "FP GZIP must produce gzip magic byte 0"); - assertEquals(data[1], (byte) 0x8b, "FP GZIP must produce gzip magic byte 1"); - } finally { - EVCacheConfig.setPropertyRepository(savedRepo); - } + EVCacheTranscoder transcoder = buildEVCacheTranscoder(null, "GZIP", null, 1); + CachedData encoded = transcoder.encode("hello world hello world hello world hello world hello world"); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x1f, "FP GZIP must produce gzip magic byte 0"); + assertEquals(data[1], (byte) 0x8b, "FP GZIP must produce gzip magic byte 1"); } @Test public void testFPAlgorithmZstd() { - DefaultSettableConfig testConfig = new DefaultSettableConfig(); - testConfig.setProperty("evcacheclient.compression.algo", "ZSTD"); - PropertyRepository savedRepo = EVCacheConfig.getInstance().getPropertyRepository(); - EVCacheConfig.setPropertyRepository(new DefaultPropertyFactory(testConfig)); - try { - EVCacheTranscoder transcoder = new EVCacheTranscoder((String) null, CachedData.MAX_SIZE, 1); - CachedData encoded = transcoder.encode("hello world hello world hello world hello world hello world"); - assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, - "COMPRESSED flag must be set"); - byte[] data = encoded.getData(); - assertEquals(data[0], (byte) 0x28, "FP ZSTD must produce zstd magic byte 0"); - assertEquals(data[1], (byte) 0xB5, "FP ZSTD must produce zstd magic byte 1"); - } finally { - EVCacheConfig.setPropertyRepository(savedRepo); - } + EVCacheTranscoder transcoder = buildEVCacheTranscoder(null, "ZSTD", null, 1); + CachedData encoded = transcoder.encode("hello world hello world hello world hello world hello world"); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x28, "FP ZSTD must produce zstd magic byte 0"); + assertEquals(data[1], (byte) 0xB5, "FP ZSTD must produce zstd magic byte 1"); } @Test public void testFPZstdLevel() { - DefaultSettableConfig testConfig = new DefaultSettableConfig(); - testConfig.setProperty("evcacheclient.compression.algo", "ZSTD"); - testConfig.setProperty("evcacheclient.compression.zstd.level", 1); - PropertyRepository savedRepo = EVCacheConfig.getInstance().getPropertyRepository(); - EVCacheConfig.setPropertyRepository(new DefaultPropertyFactory(testConfig)); - try { - EVCacheTranscoder transcoder = new EVCacheTranscoder((String) null, CachedData.MAX_SIZE, 1); - String original = "hello world hello world hello world hello world hello world"; - CachedData encoded = transcoder.encode(original); - String decoded = (String) transcoder.decode(encoded); - assertEquals(decoded, original, "FP zstd level 1 round-trip must succeed"); - } finally { - EVCacheConfig.setPropertyRepository(savedRepo); - } + EVCacheTranscoder transcoder = buildEVCacheTranscoder(null, "ZSTD", 1, 1); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = transcoder.encode(original); + String decoded = (String) transcoder.decode(encoded); + assertEquals(decoded, original, "FP zstd level 1 round-trip must succeed"); } } diff --git a/evcache-core/src/test/java/com/netflix/evcache/config/EVCacheTranscoderPropertiesTest.java b/evcache-core/src/test/java/com/netflix/evcache/config/EVCacheTranscoderPropertiesTest.java new file mode 100644 index 00000000..9fcb8ed0 --- /dev/null +++ b/evcache-core/src/test/java/com/netflix/evcache/config/EVCacheTranscoderPropertiesTest.java @@ -0,0 +1,167 @@ +package com.netflix.evcache.config; + +import static com.netflix.evcache.config.EVCacheTranscoderProperties.Key.COMPRESSION_THRESHOLD_BYTES; +import static com.netflix.evcache.config.EVCacheTranscoderProperties.Key.MAX_DATA_SIZE_BYTES; +import static org.assertj.core.api.Assertions.assertThat; + +import com.netflix.archaius.DefaultPropertyFactory; +import com.netflix.archaius.api.PropertyRepository; +import com.netflix.archaius.config.DefaultSettableConfig; + +import org.testng.annotations.Test; + +/** + * Three-level resolution tests for each key in {@link EVCacheTranscoderProperties}: + * per-app override → global default → static default. Property keys are asserted as + * string literals so a rename of a {@link EVCacheTranscoderProperties.Key} entry + * trips these tests loudly. + */ +public class EVCacheTranscoderPropertiesTest { + + private static final String APP = "MYAPP"; + private static final String BINARY_PER_APP_KEY = "MYAPP.binary.serialization.enabled"; + private static final String BINARY_GLOBAL_KEY = "default.evcache.binary.serialization.enabled"; + private static final String MAX_DATA_SIZE_PER_APP_KEY = "MYAPP.max.data.size"; + private static final String MAX_DATA_SIZE_GLOBAL_KEY = "default.evcache.max.data.size"; + private static final String COMPRESSION_PER_APP_KEY = "MYAPP.compression.threshold"; + private static final String COMPRESSION_GLOBAL_KEY = "default.evcache.compression.threshold"; + + private static PropertyRepository repo(DefaultSettableConfig cfg) { + return DefaultPropertyFactory.from(cfg); + } + + // ---- BINARY_SERIALIZATION_ENABLED ---- + + @Test + public void binarySerialization_perAppOverrideWins() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(BINARY_PER_APP_KEY, "true"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(cfg)); + assertThat(props.isBinarySerializationEnabled()).isTrue(); + } + + @Test + public void binarySerialization_globalFallbackWhenPerAppUnset() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(BINARY_GLOBAL_KEY, "true"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(cfg)); + assertThat(props.isBinarySerializationEnabled()).isTrue(); + } + + @Test + public void binarySerialization_staticDefaultWhenBothUnset() { + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(new DefaultSettableConfig())); + assertThat(props.isBinarySerializationEnabled()).isFalse(); + } + + @Test + public void binarySerialization_perAppBeatsGlobal() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(BINARY_PER_APP_KEY, "false"); + cfg.setProperty(BINARY_GLOBAL_KEY, "true"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(cfg)); + assertThat(props.isBinarySerializationEnabled()).isFalse(); + } + + @Test + public void binarySerialization_nullAppNameUsesGlobalKey() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(BINARY_GLOBAL_KEY, "true"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(null, repo(cfg)); + assertThat(props.isBinarySerializationEnabled()).isTrue(); + } + + // ---- MAX_DATA_SIZE ---- + + @Test + public void maxDataSize_perAppOverrideWins() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(MAX_DATA_SIZE_PER_APP_KEY, "12345"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(cfg)); + assertThat(props.getProperty(MAX_DATA_SIZE_BYTES, Integer.class, 999)).isEqualTo(12345); + } + + @Test + public void maxDataSize_globalFallbackWhenPerAppUnset() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(MAX_DATA_SIZE_GLOBAL_KEY, "12345"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(cfg)); + assertThat(props.getProperty(MAX_DATA_SIZE_BYTES, Integer.class, 999)).isEqualTo(12345); + } + + @Test + public void maxDataSize_staticDefaultWhenBothUnset() { + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(new DefaultSettableConfig())); + assertThat(props.getProperty(MAX_DATA_SIZE_BYTES, Integer.class, 999)).isEqualTo(999); + } + + @Test + public void maxDataSize_perAppBeatsGlobal() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(MAX_DATA_SIZE_PER_APP_KEY, "111"); + cfg.setProperty(MAX_DATA_SIZE_GLOBAL_KEY, "222"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(cfg)); + assertThat(props.getProperty(MAX_DATA_SIZE_BYTES, Integer.class, 999)).isEqualTo(111); + } + + @Test + public void maxDataSize_nullAppNameUsesGlobalKey() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(MAX_DATA_SIZE_GLOBAL_KEY, "12345"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(null, repo(cfg)); + assertThat(props.getProperty(MAX_DATA_SIZE_BYTES, Integer.class, 999)).isEqualTo(12345); + } + + // ---- COMPRESSION_THRESHOLD ---- + + @Test + public void compressionThreshold_perAppOverrideWins() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(COMPRESSION_PER_APP_KEY, "512"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(cfg)); + assertThat(props.getProperty(COMPRESSION_THRESHOLD_BYTES, Integer.class, 999)).isEqualTo(512); + } + + @Test + public void compressionThreshold_globalFallbackWhenPerAppUnset() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(COMPRESSION_GLOBAL_KEY, "512"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(cfg)); + assertThat(props.getProperty(COMPRESSION_THRESHOLD_BYTES, Integer.class, 999)).isEqualTo(512); + } + + @Test + public void compressionThreshold_staticDefaultWhenBothUnset() { + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(new DefaultSettableConfig())); + assertThat(props.getProperty(COMPRESSION_THRESHOLD_BYTES, Integer.class, 999)).isEqualTo(999); + } + + @Test + public void compressionThreshold_perAppBeatsGlobal() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(COMPRESSION_PER_APP_KEY, "111"); + cfg.setProperty(COMPRESSION_GLOBAL_KEY, "222"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(cfg)); + assertThat(props.getProperty(COMPRESSION_THRESHOLD_BYTES, Integer.class, 999)).isEqualTo(111); + } + + @Test + public void compressionThreshold_nullAppNameUsesGlobalKey() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(COMPRESSION_GLOBAL_KEY, "512"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(null, repo(cfg)); + assertThat(props.getProperty(COMPRESSION_THRESHOLD_BYTES, Integer.class, 999)).isEqualTo(512); + } +} diff --git a/evcache-core/src/test/java/com/netflix/evcache/pool/EVCacheValueSerdeTest.java b/evcache-core/src/test/java/com/netflix/evcache/pool/EVCacheValueSerdeTest.java new file mode 100644 index 00000000..02d6e271 --- /dev/null +++ b/evcache-core/src/test/java/com/netflix/evcache/pool/EVCacheValueSerdeTest.java @@ -0,0 +1,305 @@ +package com.netflix.evcache.pool; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.ByteArrayOutputStream; +import java.io.ObjectOutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; + +import com.netflix.evcache.config.EVCacheTranscoderProperties; +import org.testng.annotations.Test; + +import com.netflix.evcache.EVCacheTranscoder; + +import net.spy.memcached.CachedData; + +/** + * Pure unit tests for the compact binary serialization of {@link EVCacheValue} (the + * envelope wire format implemented inside {@link EVCacheTranscoder}), its routing through + * the transcoder, and backwards-compatibility with the legacy Java-serialized format. + * All tests go through the public {@link EVCacheTranscoder#encode(Object)} / + * {@link EVCacheTranscoder#decode(CachedData)} API — the binary codec itself is a private + * implementation detail of {@link EVCacheTranscoder}. No memcached, no DI. + */ +public class EVCacheValueSerdeTest { + + private static final int SERIALIZED = 1; // EVCacheSerializingTranscoder.SERIALIZED + private static final byte JAVA_STREAM_MAGIC_FIRST = (byte) 0xAC; + private static final byte JAVA_STREAM_MAGIC_SECOND = (byte) 0xED; + + // ---- helpers ---- + + /** Binary-enabled transcoder, compression disabled, so encoded bytes start with our magic. */ + private static EVCacheTranscoder binaryTranscoder() { + com.netflix.archaius.config.DefaultSettableConfig cfg = new com.netflix.archaius.config.DefaultSettableConfig(); + cfg.setProperty("testApp.binary.serialization.enabled", "true"); + cfg.setProperty("testApp.compression.threshold", String.valueOf(Integer.MAX_VALUE)); + + return new EVCacheTranscoder( + new EVCacheTranscoderProperties("testApp", + com.netflix.archaius.DefaultPropertyFactory.from(cfg))); + } + + /** Default transcoder (binary OFF, falls through to native Java serialization). */ + private static EVCacheTranscoder defaultTranscoder() { + return new EVCacheTranscoder(EVCacheTranscoderProperties.DEFAULT_MAX_DATA_SIZE_BYTES, Integer.MAX_VALUE); + } + + private EVCacheValue value(String key, byte[] val, int flags, long ttl, long createTime) { + return new EVCacheValue(key, val, flags, ttl, createTime); + } + + private EVCacheValue typical() { + return value("myKey", "hello world".getBytes(StandardCharsets.UTF_8), 0, 3600L, 1_700_000_000_000L); + } + + /** Serialize an object the legacy way an old client would: java.io ObjectOutputStream. */ + private byte[] javaSerialize(Object o) throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ObjectOutputStream oos = new ObjectOutputStream(baos)) { + oos.writeObject(o); + } + return baos.toByteArray(); + } + + private int javaSerializedLength(EVCacheValue v) throws Exception { + return javaSerialize(v).length; + } + + /** End-to-end round-trip via the public transcoder API with binary serialization enabled. */ + private void assertBinaryRoundTrip(EVCacheValue v) { + EVCacheTranscoder t = binaryTranscoder(); + CachedData cd = t.encode(v); + // Sanity: actually binary-encoded. + assertThat(cd.getData()[0]).isEqualTo(EVCacheValueSerde.BINARY_SERDE_MAGIC_CONSTANT_BYTE); + EVCacheValue out = (EVCacheValue) t.decode(cd); + assertThat(out).isEqualTo(v); + } + + // ---- 1. Binary round-trip across cases (via transcoder) ---- + + @Test + public void testBinaryRoundTripEmptyValue() { + assertBinaryRoundTrip(value("k", new byte[0], 0, 100L, 1L)); + } + + @Test + public void testBinaryRoundTripLargeValue() { + byte[] large = new byte[2 * 1024 * 1024]; + for (int i = 0; i < large.length; i++) { + large[i] = (byte) (i & 0xFF); + } + assertBinaryRoundTrip(value("largeKey", large, 2, 86400L, 1_700_000_000_000L)); + } + + @Test + public void testBinaryRoundTripUnicodeKey() { + assertBinaryRoundTrip(value("键🔑-é- key", + "payload".getBytes(StandardCharsets.UTF_8), 7, 60L, 42L)); + } + + @Test + public void testBinaryRoundTripZeroTtl() { + assertBinaryRoundTrip(value("zt", "v".getBytes(StandardCharsets.UTF_8), 1, 0L, 42L)); + } + + @Test + public void testBinaryRoundTripNegativeCreateTime() { + assertBinaryRoundTrip(value("nct", "v".getBytes(StandardCharsets.UTF_8), 1, 60L, -987654321L)); + } + + @Test + public void testBinaryRoundTripMaxCreateTime() { + assertBinaryRoundTrip(value("mct", "v".getBytes(StandardCharsets.UTF_8), 1, 60L, Long.MAX_VALUE)); + } + + @Test + public void testBinaryRoundTripMinFlags() { + assertBinaryRoundTrip(value("minf", "v".getBytes(StandardCharsets.UTF_8), Integer.MIN_VALUE, 60L, 42L)); + } + + // ---- 2. Transcoder produces expected wire shape (binary mode) ---- + + @Test + public void testTranscoderBinaryWireShape() { + EVCacheTranscoder t = binaryTranscoder(); + EVCacheValue v = typical(); + + CachedData cd = t.encode(v); + + // SERIALIZED flag must be set so decode routes through deserialize(). + assertThat(cd.getFlags() & SERIALIZED).isNotZero(); + // Binary envelope marker present (no compression interfering). + assertThat(cd.getData()[0]).isEqualTo(EVCacheValueSerde.BINARY_SERDE_MAGIC_CONSTANT_BYTE); + // Byte index 1 is the reserved/version byte, currently always 0x00. + assertThat(cd.getData()[1]).isEqualTo((byte) 0x00); + + Object out = t.decode(cd); + assertThat(out).isInstanceOf(EVCacheValue.class); + assertThat(out).isEqualTo(v); + } + + // ---- 3. Default transcoder writes Java, but decode reads both formats ---- + + @Test + public void testTranscoderDefaultProducesJavaAndReadsBoth() { + EVCacheTranscoder t = defaultTranscoder(); + EVCacheValue v = typical(); + + CachedData cd = t.encode(v); + + // Java serialization stream magic is 0xAC 0xED. + byte[] data = cd.getData(); + assertThat(data[0]).isEqualTo(JAVA_STREAM_MAGIC_FIRST); + assertThat(data[1]).isEqualTo(JAVA_STREAM_MAGIC_SECOND); + // SERIALIZED flag still set. + assertThat(cd.getFlags() & SERIALIZED).isNotZero(); + + // Dual-format read: default-Java write decodes back to an equal EVCacheValue. + Object out = t.decode(cd); + assertThat(out).isInstanceOf(EVCacheValue.class); + assertThat(out).isEqualTo(v); + } + + // ---- 4. Backwards-compat: new client reads legacy Java-serialized bytes ---- + + @Test + public void testBackwardsCompatLegacyJavaSerialized() throws Exception { + EVCacheValue v = typical(); + byte[] javaBytes = javaSerialize(v); + + // Sanity: legacy bytes start with the Java stream header, not our binary magic. + assertThat(javaBytes[0]).isEqualTo(JAVA_STREAM_MAGIC_FIRST); + assertThat(javaBytes[0]).isNotEqualTo(EVCacheValueSerde.BINARY_SERDE_MAGIC_CONSTANT_BYTE); + + CachedData cd = new CachedData(SERIALIZED, javaBytes, CachedData.MAX_SIZE); + Object out = defaultTranscoder().decode(cd); + + assertThat(out).isInstanceOf(EVCacheValue.class); + assertThat(out).isEqualTo(v); + } + + // ---- 5. Non-EVCacheValue passthrough (arbitrary Java objects still use Java serde) ---- + + @Test + public void testNonEVCacheValuePassthrough() { + EVCacheTranscoder t = binaryTranscoder(); // even with binary on, non-EVCacheValue stays Java + ArrayList list = new ArrayList<>(); + list.add("a"); + list.add("b"); + list.add("c"); + + CachedData cd = t.encode(list); + Object out = t.decode(cd); + + assertThat(out).isEqualTo(list); + // Routed through generic Java serialization, not the binary envelope. + assertThat(cd.getFlags() & SERIALIZED).isNotZero(); + assertThat(cd.getData()[0]).isEqualTo(JAVA_STREAM_MAGIC_FIRST); + } + + // ---- 6. Size win: binary smaller than Java for a representative item ---- + + @Test + public void testBinaryIsSmallerThanJava() throws Exception { + EVCacheValue v = typical(); + int binaryLen = binaryTranscoder().encode(v).getData().length; + int javaLen = javaSerializedLength(v); + assertThat(binaryLen).isLessThan(javaLen); + } + + // ---- 7. Malformed binary input is logged in EVCacheValueSerde and decodes to null ---- + // + // EVCacheValueSerde.deserialize warn-logs the corruption (field + truncated hex) and returns + // null. Callers see a cache miss rather than a thrown exception, matching the resilience + // contract of BaseSerializingTranscoder. + + @Test + public void testDecodeTruncatedBinaryReturnsNull() { + byte[] full = binaryTranscoder().encode(typical()).getData(); + byte[] truncated = Arrays.copyOf(full, 3); + CachedData cd = new CachedData(SERIALIZED, truncated, CachedData.MAX_SIZE); + assertThat(defaultTranscoder().decode(cd)).isNull(); + } + + @Test + public void testDecodeBinaryWithBogusKeyLengthReturnsNull() { + // Magic + reserved + wildly oversized keyLength. Bounds check rejects. + byte[] bytes = new byte[2 + Integer.BYTES]; + bytes[0] = EVCacheValueSerde.BINARY_SERDE_MAGIC_CONSTANT_BYTE; + bytes[1] = 0x00; + ByteBuffer bb = ByteBuffer.wrap(bytes).order(ByteOrder.BIG_ENDIAN); + bb.putInt(2, 0x7FFFFFFF); + CachedData cd = new CachedData(SERIALIZED, bytes, CachedData.MAX_SIZE); + assertThat(defaultTranscoder().decode(cd)).isNull(); + } + + @Test + public void testDecodeBinaryWithNegativeKeyLengthReturnsNull() { + byte[] bytes = new byte[2 + Integer.BYTES]; + bytes[0] = EVCacheValueSerde.BINARY_SERDE_MAGIC_CONSTANT_BYTE; + bytes[1] = 0x00; + ByteBuffer bb = ByteBuffer.wrap(bytes).order(ByteOrder.BIG_ENDIAN); + bb.putInt(2, -1); + CachedData cd = new CachedData(SERIALIZED, bytes, CachedData.MAX_SIZE); + assertThat(defaultTranscoder().decode(cd)).isNull(); + } + + // ---- 8. Forward compatibility trip-wire: pinned v0 payload must always decode ---- + // + // If this test starts failing after a change to EVCacheValueSerde.deserialize(), someone + // likely added a required field without the `buffer.hasRemaining()` guard. See the + // "Additive optional" section of EVCacheValueSerde's Javadoc — a future reader must be + // able to decode the v0 payload below (which an old writer would have produced) for as + // long as items written by old writers can still be in any cache. + // + // The bytes here are intentionally FROZEN. Do not update them when adding fields. + @Test + public void testV0PayloadDecodesAsOptionalAdditiveFieldTripWire() { + byte[] v0Bytes = { + (byte) 0x0C, // magic + (byte) 0x00, // reserved/version + 0x00, 0x00, 0x00, 0x01, // keyLength = 1 + (byte) 'k', + 0x00, 0x00, 0x00, 0x01, // valueLength = 1 + 0x76, // value byte + 0x00, 0x00, 0x00, 0x01, // flags = 1 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3C, // ttl = 60 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2A, // createTime = 42 + }; + CachedData cd = new CachedData(SERIALIZED, v0Bytes, CachedData.MAX_SIZE); + Object out = defaultTranscoder().decode(cd); + + EVCacheValue expected = new EVCacheValue("k", new byte[] {0x76}, 1, 60L, 42L); + assertThat(out) + .as("Pinned v0 payload must decode cleanly. If it doesn't, a required field was " + + "likely added to deserialize() without the buffer.hasRemaining() guard. " + + "See EVCacheValueSerde Javadoc 'Additive optional'.") + .isEqualTo(expected); + } + + // ---- 9. Forward compatibility: newer-writer extension bytes past createTime ---- + // + // A writer that adds new optional fields appends them after createTime. An older reader + // (this one) reads its known fields, leaves the extension bytes unread, and returns the + // EVCacheValue it does know how to decode — NOT a corruption event. + + @Test + public void testDecodeBinaryWithFutureExtensionFieldsIsForwardCompat() { + EVCacheValue v = typical(); + byte[] validBytes = binaryTranscoder().encode(v).getData(); + + // Append 3 extension bytes past the end of the v0 envelope — what a future writer + // would do. End of envelope is implicit at bytes.length, no header to update. + byte[] withExtension = Arrays.copyOf(validBytes, validBytes.length + 3); + + CachedData cd = new CachedData(SERIALIZED, withExtension, CachedData.MAX_SIZE); + Object out = defaultTranscoder().decode(cd); + assertThat(out).isInstanceOf(EVCacheValue.class); + assertThat(out).isEqualTo(v); + } +} diff --git a/evcache-core/src/test/java/test-suite.xml b/evcache-core/src/test/java/test-suite.xml index 194ea07e..39ab6875 100644 --- a/evcache-core/src/test/java/test-suite.xml +++ b/evcache-core/src/test/java/test-suite.xml @@ -3,6 +3,8 @@ + +