diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java index d6280faf38be9..28b890665c221 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java @@ -107,7 +107,7 @@ public void setup() { mapping.put("field" + i, new EsField("field-" + i, TEXT, emptyMap(), true, EsField.TimeSeriesFieldType.NONE)); } - var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD), Map.of(), Map.of(), Map.of()); + var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD), Map.of(), Map.of()); var functionRegistry = new EsqlFunctionRegistry(); parser = new EsqlParser(new EsqlConfig(functionRegistry)); diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/esql/ViewResolutionBenchmarkBase.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/esql/ViewResolutionBenchmarkBase.java index 048945c12dbd9..0c16e1fb83244 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/esql/ViewResolutionBenchmarkBase.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/esql/ViewResolutionBenchmarkBase.java @@ -168,7 +168,7 @@ public void setup() { String name = "col" + i; mapping.put(name, new EsField(name, KEYWORD, emptyMap(), true, EsField.TimeSeriesFieldType.NONE)); } - EsIndex esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD), Map.of(), Map.of(), Map.of()); + EsIndex esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD), Map.of(), Map.of()); Configuration config = new Configuration( DateUtils.UTC, diff --git a/server/src/main/resources/transport/definitions/referable/compact_multi_type_es_field.csv b/server/src/main/resources/transport/definitions/referable/compact_multi_type_es_field.csv new file mode 100644 index 0000000000000..1e122475cc639 --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/compact_multi_type_es_field.csv @@ -0,0 +1 @@ +9366000 diff --git a/server/src/main/resources/transport/upper_bounds/9.5.csv b/server/src/main/resources/transport/upper_bounds/9.5.csv index e16cac0347b3d..320d7d7ab2161 100644 --- a/server/src/main/resources/transport/upper_bounds/9.5.csv +++ b/server/src/main/resources/transport/upper_bounds/9.5.csv @@ -1 +1 @@ -inference_api_audio_video_pdf_support,9365000 +compact_multi_type_es_field,9366000 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/TestAnalyzer.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/TestAnalyzer.java index 1378a991402f9..4f5a2a02d6322 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/TestAnalyzer.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/TestAnalyzer.java @@ -154,8 +154,7 @@ public TestAnalyzer addNoFieldsIndex() { Map.of(), Map.of(noFieldsIndexName, IndexMode.STANDARD), Map.of("", List.of(noFieldsIndexName)), - Map.of("", List.of(noFieldsIndexName)), - Map.of() + Map.of("", List.of(noFieldsIndexName)) ); addIndex(noFieldsIndexName, IndexResolution.valid(noFieldsIndex)); return this; @@ -781,7 +780,7 @@ public AnalyzerContext buildContext() { */ public static IndexResolution loadMapping(String resource, String indexName, IndexMode indexMode) { return IndexResolution.valid( - new EsIndex(indexName, EsqlTestUtils.loadMapping(resource), Map.of(indexName, indexMode), Map.of(), Map.of(), Map.of()) + new EsIndex(indexName, EsqlTestUtils.loadMapping(resource), Map.of(indexName, indexMode), Map.of(), Map.of()) ); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index 903498dc87ef4..07e468bf88b64 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -537,6 +537,22 @@ sample_data | Connected to 10.1.0.2 sample_data | Connected to 10.1.0.1 ; +partiallyUnmappedSmallNumericFieldIsWidened +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM apps_short, partial_mapping_sample_data +| KEEP id, name +| SORT name DESC NULLS LAST +| LIMIT 3 +; + +id:integer | name:keyword +14 | mmmmm +13 | lllll +11 | kkkkk +; + fieldIsPartiallyUnmappedAndRenamedMultiIndex required_capability: optional_fields_v5 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 25ad697aed6bc..4fa6c4aa95824 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -53,11 +53,13 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.CompactMultiTypeEsField; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; -import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; +import org.elasticsearch.xpack.esql.core.type.TypeConflictField; +import org.elasticsearch.xpack.esql.core.type.UnionTypeEsField; import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; import org.elasticsearch.xpack.esql.core.util.CollectionUtils; import org.elasticsearch.xpack.esql.core.util.Holder; @@ -181,7 +183,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeSet; import java.util.function.Function; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -352,10 +353,6 @@ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexR var attributes = mappingAsAttributes(plan.source(), esIndex.mapping()); attributes.addAll(metadata.stream().map(NamedExpression::toAttribute).toList()); - if (context.unmappedResolution() == UnmappedResolution.LOAD) { - loadPartiallyUnmappedFields(attributes, esIndex); - } - return new EsRelation( plan.source(), esIndex.name(), @@ -367,34 +364,6 @@ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexR ); } - /** - * When {@code SET unmapped_fields="load"}, convert partially-mapped fields so they can be used across indices where they - * may not exist: - * - */ - private static void loadPartiallyUnmappedFields(List attributes, EsIndex esIndex) { - for (int i = 0; i < attributes.size(); i++) { - if (attributes.get(i) instanceof FieldAttribute fa && isPartiallyUnmappedRegularField(fa, esIndex)) { - if (fa.dataType() == KEYWORD) { - attributes.set(i, ResolveRefs.insistKeyword(fa)); - } else { - attributes.set(i, ResolveRefs.invalidInsistAttribute(fa, esIndex)); - } - } - } - } - - private static boolean isPartiallyUnmappedRegularField(FieldAttribute fa, EsIndex esIndex) { - // We ignore proper subclasses of FieldAttribute; these represent unsupported or special attributes. - return fa.getClass().equals(FieldAttribute.class) && esIndex.isPartiallyUnmappedField(fa.fieldName().string()); - } - private List resolveMetadata(List metadata, AnalyzerContext context) { LinkedHashMap resolved = new LinkedHashMap<>(); Set allTags = null; @@ -690,7 +659,7 @@ protected LogicalPlan rule(LogicalPlan plan, AnalyzerContext context) { case MvExpand p -> resolveMvExpand(p, childrenOutput); case Lookup l -> resolveLookup(l, childrenOutput); case LookupJoin j -> resolveLookupJoin(j, context); - case Insist i -> resolveInsist(i, childrenOutput, context); + case Insist i -> resolveInsist(i, childrenOutput); case Fuse fuse -> resolveFuse(fuse, childrenOutput); case Rerank r -> resolveRerank(r, childrenOutput, context); case PromqlCommand promql -> resolvePromql(promql, childrenOutput); @@ -1220,60 +1189,26 @@ private List resolveUsingColumns(List cols, List childrenOutput, AnalyzerContext context) { + private LogicalPlan resolveInsist(Insist insist, List childrenOutput) { List list = new ArrayList<>(); - List resolutions = collectIndexResolutions(insist, context); for (Attribute a : insist.insistedAttributes()) { - list.add(resolveInsistAttribute(a, childrenOutput, resolutions)); + list.add(resolveInsistAttribute(a, childrenOutput)); } return insist.withAttributes(list); } - private static List collectIndexResolutions(LogicalPlan plan, AnalyzerContext context) { - List resolutions = new ArrayList<>(); - plan.forEachDown(EsRelation.class, e -> { - var resolution = context.indexResolution().get(new IndexPattern(e.source(), e.indexPattern())); - if (resolution != null) { - resolutions.add(resolution); - } - }); - return resolutions; - } - - private Attribute resolveInsistAttribute(Attribute attribute, List childrenOutput, List indices) { + private Attribute resolveInsistAttribute(Attribute attribute, List childrenOutput) { Attribute resolvedCol = maybeResolveAttribute((UnresolvedAttribute) attribute, childrenOutput); // Field isn't mapped anywhere. if (resolvedCol instanceof UnresolvedAttribute) { return insistKeyword(attribute); } - // Field is partially unmapped. - // TODO: Should the check for partially unmapped fields be done specific to each sub-query in a fork? - if (resolvedCol instanceof FieldAttribute fa && indices.stream().anyMatch(r -> r.get().isPartiallyUnmappedField(fa.name()))) { - // NOTE: We use indices.getFirst() here as a temporary solution. INSIST will be removed after load is in GA anyway. - return fa.dataType() == KEYWORD ? insistKeyword(fa) : invalidInsistAttribute(fa, indices.getFirst().get()); - } - - // Either the field is mapped everywhere and we can just use the resolved column, or the INSIST clause isn't on top of a FROM - // clause—for example, it might be on top of a ROW clause—so the verifier will catch it and fail. + // Partially unmapped fields are already wrapped during index resolution: + // keyword → PotentiallyUnmappedKeywordEsField, non-keyword → InvalidMappedField.potentiallyUnmapped. return resolvedCol; } - static FieldAttribute invalidInsistAttribute(FieldAttribute fa, EsIndex esIndex) { - InvalidMappedField field = InvalidMappedField.potentiallyUnmapped(fa.field().getName(), getTypesToIndices(fa, esIndex)); - return new FieldAttribute(fa.source(), fa.parentName(), fa.qualifier(), fa.name(), field); - } - - private static Map> getTypesToIndices(FieldAttribute fa, EsIndex esIndex) { - if (fa.field() instanceof InvalidMappedField imf) { - return imf.getTypesToIndices(); - } - // Field isn't currently invalid, meaning it's mapped to a single type in all the indices where it's actually mapped. - TreeSet indicesWithField = new TreeSet<>(esIndex.concreteQualifiedIndices()); - indicesWithField.removeAll(esIndex.getUnmappedIndices(fa.name())); - return Map.of(fa.dataType().typeName(), indicesWithField); - } - public static FieldAttribute insistKeyword(Attribute attribute) { return new FieldAttribute( attribute.source(), @@ -2310,23 +2245,27 @@ private static Expression processVectorFunction( * Any fields which could not be resolved by conversion functions will be converted to UnresolvedAttribute instances in a later rule * (See {@link UnionTypesCleanup} below). */ - private static class ResolveUnionTypes extends Rule { + private static class ResolveUnionTypes extends ParameterizedRule { record TypeResolutionKey(String fieldName, DataType fieldType) {} @Override - public LogicalPlan apply(LogicalPlan plan) { + public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { List unionFieldAttributes = new ArrayList<>(); - return plan.transformUp(LogicalPlan.class, p -> p.childrenResolved() == false ? p : doRule(p, unionFieldAttributes)); + return plan.transformUp(LogicalPlan.class, p -> p.childrenResolved() ? doRule(p, unionFieldAttributes, context) : p); } - private LogicalPlan doRule(LogicalPlan plan, List unionFieldAttributes) { + private static LogicalPlan doRule( + LogicalPlan plan, + List unionFieldAttributes, + AnalyzerContext context + ) { Holder alreadyAddedUnionFieldAttributes = new Holder<>(unionFieldAttributes.size()); // Collect field attributes from previous runs if (plan instanceof EsRelation rel) { unionFieldAttributes.clear(); for (Attribute attr : rel.output()) { - if (attr instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField && fa.synthetic()) { + if (attr instanceof FieldAttribute fa && fa.field() instanceof UnionTypeEsField && fa.synthetic()) { unionFieldAttributes.add(fa.ignoreId()); } } @@ -2336,7 +2275,7 @@ private LogicalPlan doRule(LogicalPlan plan, List u // Replace the entire convert function with a new FieldAttribute (containing type conversion knowledge) plan = plan.transformExpressionsOnly(e -> { if (e instanceof ConvertFunction convert) { - return resolveConvertFunction(convert, unionFieldAttributes); + return resolveConvertFunction(convert, unionFieldAttributes, context); } return e; }); @@ -2389,9 +2328,13 @@ private static LogicalPlan addGeneratedFieldsToEsRelations(LogicalPlan plan, Lis return res; } - private Expression resolveConvertFunction(ConvertFunction convert, List unionFieldAttributes) { + private static Expression resolveConvertFunction( + ConvertFunction convert, + List unionFieldAttributes, + AnalyzerContext context + ) { Expression convertExpression = (Expression) convert; - if (convert.field() instanceof FieldAttribute fa && fa.field() instanceof InvalidMappedField imf) { + if (convert.field() instanceof FieldAttribute fa && fa.field() instanceof TypeConflictField imf) { HashMap typeResolutions = new HashMap<>(); Set supportedTypes = convert.supportedTypes(); if (convert instanceof FoldablesConvertFunction fcf) { @@ -2414,62 +2357,45 @@ private Expression resolveConvertFunction(ConvertFunction convert, List supportedTypes = convert.supportedTypes(); - if (supportedTypes.contains(fa.dataType()) && canConvertOriginalTypes(mtf, supportedTypes)) { - // Build the mapping between index name and conversion expressions - Map indexToConversionExpressions = new HashMap<>(); - for (Map.Entry entry : mtf.getIndexToConversionExpressions().entrySet()) { - String indexName = entry.getKey(); - AbstractConvertFunction originalConversionFunction = (AbstractConvertFunction) entry.getValue(); - Expression originalField = originalConversionFunction.field(); - Expression newConvertFunction = convertExpression.replaceChildren(Collections.singletonList(originalField)); - indexToConversionExpressions.put(indexName, newConvertFunction); - } + if (supportedTypes.contains(fa.dataType()) && canConvertOriginalTypes(unionTypeEsField, supportedTypes)) { // The only code that creates MultiTypeEsField with synthetic=false (reaching this branch) is // DateMillisToNanosInEsRelation, which runs in the "Initialize" batch before ResolveUnmapped. At that point, // unmapped fields haven't been detected yet, so potentiallyUnmappedExpression is always null. - if (mtf.getPotentiallyUnmappedExpression() != null) { + if (((UnionTypeEsField) fa.field()).getUnmappedConversionExpression() != null) { throw new IllegalStateException("Unexpected potentially unmapped expression for [" + fa.fieldName() + "]"); } - MultiTypeEsField multiTypeEsField = new MultiTypeEsField( - fa.fieldName().string(), - convertExpression.dataType(), - false, - indexToConversionExpressions, - fa.field().getTimeSeriesFieldType(), - null - ); - return createIfDoesNotAlreadyExist(fa, multiTypeEsField, unionFieldAttributes); + return createIfDoesNotAlreadyExist(fa, unionTypeEsField.rewrapWithCast(convertExpression), unionFieldAttributes); } } else if (convert.field() instanceof AbstractConvertFunction subConvert) { return convertExpression.replaceChildren( - Collections.singletonList(resolveConvertFunction(subConvert, unionFieldAttributes)) + singletonList(resolveConvertFunction(subConvert, unionFieldAttributes, context)) ); } return convertExpression; } - private Expression createIfDoesNotAlreadyExist( + private static Expression createIfDoesNotAlreadyExist( FieldAttribute fa, - MultiTypeEsField resolvedField, + EsField resolvedField, List unionFieldAttributes ) { // Generate new ID for the field and suffix it with the data type to maintain unique attribute names. @@ -2496,13 +2422,14 @@ private Expression createIfDoesNotAlreadyExist( } } - private static MultiTypeEsField resolvedMultiTypeEsField( + static EsField resolvedMultiTypeEsField( FieldAttribute fa, Map typeResolutions, - @Nullable Expression potentiallyUnmappedConversion + @Nullable Expression potentiallyUnmappedConversion, + AnalyzerContext context ) { Map typesToConversionExpressions = new HashMap<>(); - InvalidMappedField imf = (InvalidMappedField) fa.field(); + TypeConflictField imf = (TypeConflictField) fa.field(); imf.getTypesToIndices().forEach((typeName, indexNames) -> { DataType type = DataType.fromTypeName(typeName); TypeResolutionKey key = new TypeResolutionKey(fa.name(), type); @@ -2510,13 +2437,27 @@ private static MultiTypeEsField resolvedMultiTypeEsField( typesToConversionExpressions.put(typeName, typeResolutions.get(key)); } }); - return MultiTypeEsField.resolveFrom(imf, typesToConversionExpressions) - .withPotentiallyUnmappedExpression(potentiallyUnmappedConversion); + return buildMultiTypeEsField(imf, typesToConversionExpressions, potentiallyUnmappedConversion, context); } - private static boolean canConvertOriginalTypes(MultiTypeEsField multiTypeEsField, Set supportedTypes) { - return multiTypeEsField.getIndexToConversionExpressions() - .values() + /** + * Picks between the legacy {@link MultiTypeEsField} and the new {@link CompactMultiTypeEsField} based on the cluster minimum + * transport version, so that newly-built plans remain deserializable on older nodes. + */ + private static EsField buildMultiTypeEsField( + TypeConflictField imf, + Map typesToConversionExpressions, + @Nullable Expression unmappedConversionExpression, + AnalyzerContext context + ) { + return context.minimumVersion().supports(CompactMultiTypeEsField.CompactMultiTypeEsField) + ? CompactMultiTypeEsField.resolveFrom(imf, typesToConversionExpressions, unmappedConversionExpression) + : MultiTypeEsField.resolveFrom(imf, typesToConversionExpressions) + .withPotentiallyUnmappedExpression(unmappedConversionExpression); + } + + private static boolean canConvertOriginalTypes(UnionTypeEsField unionTypeEsField, Set supportedTypes) { + return unionTypeEsField.getConversionExpressions() .stream() .allMatch( e -> e instanceof AbstractConvertFunction convertFunction @@ -2524,7 +2465,7 @@ private static boolean canConvertOriginalTypes(MultiTypeEsField multiTypeEsField ); } - private static Expression typeSpecificConvert(ConvertFunction convert, Source source, DataType type, InvalidMappedField mtf) { + private static Expression typeSpecificConvert(ConvertFunction convert, Source source, DataType type, TypeConflictField mtf) { EsField field = new EsField(mtf.getName(), type, mtf.getProperties(), mtf.isAggregatable(), mtf.getTimeSeriesFieldType()); FieldAttribute originalFieldAttr = (FieldAttribute) convert.field(); FieldAttribute resolvedAttr = new FieldAttribute( @@ -2584,7 +2525,7 @@ public LogicalPlan apply(LogicalPlan plan) { */ private static Attribute cleanTypeConflicts(FieldAttribute fa) { EsField field = fa.field(); - if (field instanceof InvalidMappedField imf && imf.isPotentiallyUnmapped() && imf.types().size() == 1) { + if (field instanceof TypeConflictField imf && imf.isPotentiallyUnmapped() && imf.types().size() == 1) { DataType type = imf.types().iterator().next(); var restoredField = new EsField(imf.getName(), type, imf.getProperties(), false, imf.getTimeSeriesFieldType()); // TODO: add test where not passing on the parent name fails the test @@ -2630,7 +2571,7 @@ public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { return relation; } return relation.transformExpressionsUp(FieldAttribute.class, f -> { - if (f.field() instanceof InvalidMappedField imf && allDates(context, imf)) { + if (f.field() instanceof TypeConflictField imf && allDates(context, imf)) { HashMap typeResolutions = new HashMap<>(); var convert = new ToDateNanos(f.source(), f, context.configuration()); imf.types().forEach(type -> typeResolutions(f, convert, type, imf, typeResolutions)); @@ -2638,7 +2579,7 @@ public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { // potentiallyUnmapped fields. This assertion guards against future changes breaking that invariant. assert imf.isPotentiallyUnmapped() == false : "Unexpected potentially unmapped field [" + imf.getName() + "] in DateMillisToNanosInEsRelation"; - var resolvedField = ResolveUnionTypes.resolvedMultiTypeEsField(f, typeResolutions, null); + var resolvedField = ResolveUnionTypes.resolvedMultiTypeEsField(f, typeResolutions, null, context); return new FieldAttribute( f.source(), f.parentName(), @@ -2655,7 +2596,7 @@ public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { }); } - private static boolean allDates(AnalyzerContext context, InvalidMappedField imf) { + private static boolean allDates(AnalyzerContext context, TypeConflictField imf) { if (imf.types().stream().allMatch(DataType::isDate) == false) { return false; } @@ -2672,7 +2613,7 @@ private static void typeResolutions( FieldAttribute fieldAttribute, ConvertFunction convert, DataType type, - InvalidMappedField imf, + TypeConflictField imf, HashMap typeResolutions ) { ResolveUnionTypes.TypeResolutionKey key = new ResolveUnionTypes.TypeResolutionKey(fieldAttribute.name(), type); @@ -2685,19 +2626,19 @@ private static void typeResolutions( * are aggregate metric double + any combination of numerics, implicitly cast them to the same type: aggregate metric * double for count, and double for min, max, and sum. Avg gets replaced with its surrogate (Div(Sum, Count)) */ - private static class ImplicitCastAggregateMetricDoubles extends Rule { + private static class ImplicitCastAggregateMetricDoubles extends ParameterizedRule { private boolean isTimeSeries = false; @Override - public LogicalPlan apply(LogicalPlan plan) { + public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { Holder indexMode = new Holder<>(IndexMode.STANDARD); plan.forEachUp(EsRelation.class, esRelation -> { indexMode.set(esRelation.indexMode()); }); isTimeSeries = indexMode.get() == IndexMode.TIME_SERIES; - return plan.transformUp(LogicalPlan.class, this::doRule); + return plan.transformUp(LogicalPlan.class, p -> doRule(p, context)); } - private LogicalPlan doRule(LogicalPlan plan) { + private LogicalPlan doRule(LogicalPlan plan, AnalyzerContext context) { if (plan instanceof EsRelation || plan instanceof Project || plan.childrenResolved() == false) { return plan; } @@ -2706,17 +2647,17 @@ private LogicalPlan doRule(LogicalPlan plan) { var newPlan = plan.transformExpressionsOnly(AggregateFunction.class, aggFunc -> { Expression child; if (aggFunc.field() instanceof ToAggregateMetricDouble toAMD) { - child = tryToTransformFunction(aggFunc, toAMD.field(), aborted, unionFields); + child = tryToTransformFunction(aggFunc, toAMD.field(), aborted, unionFields, context); } else { - child = tryToTransformFunction(aggFunc, aggFunc.field(), aborted, unionFields); + child = tryToTransformFunction(aggFunc, aggFunc.field(), aborted, unionFields, context); } return child; }).transformExpressionsOnly(EsqlBinaryComparison.class, comparison -> { Expression left = comparison.left(); Expression right = comparison.right(); Holder modified = new Holder<>(Boolean.FALSE); - left = tryToTransformBinaryComparison(comparison, left, modified, unionFields); - right = tryToTransformBinaryComparison(comparison, right, modified, unionFields); + left = tryToTransformBinaryComparison(comparison, left, modified, unionFields, context); + right = tryToTransformBinaryComparison(comparison, right, modified, unionFields, context); if (modified.get() == false) { return comparison; } @@ -2732,9 +2673,10 @@ private Expression tryToTransformBinaryComparison( EsqlBinaryComparison comparison, Expression original, Holder modified, - Map unionFields + Map unionFields, + AnalyzerContext context ) { - if (original instanceof FieldAttribute fa && fa.field() instanceof InvalidMappedField imf && canBeCasted(imf)) { + if (original instanceof FieldAttribute fa && fa.field() instanceof TypeConflictField imf && canBeCasted(imf)) { Map typeConverters = new HashMap<>(); for (DataType type : imf.types()) { ConvertFunction convert = type == AGGREGATE_METRIC_DOUBLE @@ -2750,7 +2692,7 @@ private Expression tryToTransformBinaryComparison( fa.parentName(), fa.qualifier(), newName, - MultiTypeEsField.resolveFrom(imf, typeConverters), + ResolveUnionTypes.buildMultiTypeEsField(imf, typeConverters, null, context), fa.nullable(), null, true @@ -2762,7 +2704,7 @@ private Expression tryToTransformBinaryComparison( return original; } - private static boolean canBeCasted(InvalidMappedField imf) { + private static boolean canBeCasted(TypeConflictField imf) { return imf.types().contains(AGGREGATE_METRIC_DOUBLE) && imf.types().stream().allMatch(f -> f == AGGREGATE_METRIC_DOUBLE || f.isNumeric()); } @@ -2771,9 +2713,10 @@ private Expression tryToTransformFunction( AggregateFunction aggFunc, Expression field, Holder aborted, - Map unionFields + Map unionFields, + AnalyzerContext context ) { - if (field instanceof FieldAttribute fa && fa.field() instanceof InvalidMappedField imf) { + if (field instanceof FieldAttribute fa && fa.field() instanceof TypeConflictField imf) { if (canBeCasted(imf) == false) { aborted.set(Boolean.TRUE); return aggFunc; @@ -2803,7 +2746,7 @@ private Expression tryToTransformFunction( fa.parentName(), fa.qualifier(), newName, - MultiTypeEsField.resolveFrom(imf, typeConverters), + ResolveUnionTypes.buildMultiTypeEsField(imf, typeConverters, null, context), fa.nullable(), null, true @@ -2823,7 +2766,7 @@ private Expression tryToTransformFunction( return aggFunc; } - private Map typeConverters(AggregateFunction aggFunc, FieldAttribute fa, InvalidMappedField mtf) { + private Map typeConverters(AggregateFunction aggFunc, FieldAttribute fa, TypeConflictField mtf) { var metric = getMetric(aggFunc, isTimeSeries); Map typeConverter = new HashMap<>(); for (DataType type : mtf.types()) { @@ -2844,7 +2787,7 @@ private Map typeConverters(AggregateFunction aggFunc, FieldA return typeConverter; } - private Expression countConvert(UnaryScalarFunction convert, Source source, DataType type, InvalidMappedField imf) { + private Expression countConvert(UnaryScalarFunction convert, Source source, DataType type, TypeConflictField imf) { EsField field = new EsField(imf.getName(), type, imf.getProperties(), imf.isAggregatable(), imf.getTimeSeriesFieldType()); FieldAttribute originalFieldAttr = (FieldAttribute) convert.field(); FieldAttribute resolvedAttr = new FieldAttribute( @@ -3367,7 +3310,7 @@ private static List collectIncompatibleTypes(int columnIndex, List dataTypes = new ArrayList<>(); for (List out : outputs) { Attribute attr = out.get(columnIndex); - if (attr instanceof FieldAttribute fa && fa.field() instanceof InvalidMappedField imf) { + if (attr instanceof FieldAttribute fa && fa.field() instanceof TypeConflictField imf) { dataTypes.addAll(imf.types().stream().map(DataType::typeName).toList()); } else { dataTypes.add(attr.dataType().typeName()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 0146025388298..bd058c121b6d6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.analysis; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.license.XPackLicenseState; @@ -31,8 +32,10 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.esql.core.tree.Node; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; +import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; +import org.elasticsearch.xpack.esql.core.type.TypeConflictField; +import org.elasticsearch.xpack.esql.core.type.UnionTypeEsField; import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.expression.function.TimestampAware; @@ -44,7 +47,6 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; -import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.index.IndexResolution; import org.elasticsearch.xpack.esql.plan.IndexPattern; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; @@ -613,13 +615,12 @@ private static AttributeSet partiallyUnmappedNonKeywords(LogicalPlan plan, Map { IndexResolution indexResolution = indexResolutions.get(new IndexPattern(relation.source(), relation.indexPattern())); if (indexResolution != null && indexResolution.isValid()) { - EsIndex index = indexResolution.get(); + Set punkFieldNames = collectPotentiallyUnmappedNonKeywords(indexResolution.get().mapping()); for (Attribute attr : relation.output()) { - if (attr instanceof FieldAttribute fa - && index.isPartiallyUnmappedField(fa.fieldName().string()) - && fa.dataType() != DataType.KEYWORD // punk_field::long is fine; in this case, the FieldAttribute contains a MultiTypeEsField with the conversions. - && fa.field() instanceof MultiTypeEsField == false) { + if (attr instanceof FieldAttribute fa + && punkFieldNames.contains(fa.fieldName().string()) + && fa.field() instanceof UnionTypeEsField == false) { punks.add(fa); } } @@ -629,6 +630,29 @@ private static AttributeSet partiallyUnmappedNonKeywords(LogicalPlan plan, Map collectPotentiallyUnmappedNonKeywords(Map mapping) { + HashSet result = new HashSet<>(); + collectPotentiallyUnmappedNonKeywords(mapping, null, result); + return result; + } + + private static void collectPotentiallyUnmappedNonKeywords( + Map mapping, + @Nullable String prefix, + Set aggregator + ) { + for (Map.Entry entry : mapping.entrySet()) { + String name = prefix == null ? entry.getKey() : prefix + "." + entry.getKey(); + EsField field = entry.getValue(); + if (field instanceof TypeConflictField imf && imf.isPotentiallyUnmapped()) { + aggregator.add(name); + } + if (field.getProperties() != null && field.getProperties().isEmpty() == false) { + collectPotentiallyUnmappedNonKeywords(field.getProperties(), name, aggregator); + } + } + } + private void licenseCheck(LogicalPlan plan, Failures failures) { Consumer> licenseCheck = n -> { if (n instanceof LicenseAware la && la.licenseCheck(licenseState) == false) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java index c0d274bb51ece..da59e6b659a36 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java @@ -8,7 +8,7 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; +import org.elasticsearch.xpack.esql.core.type.TypeConflictField; import java.util.ArrayList; import java.util.Collection; @@ -40,7 +40,7 @@ public static List asAttributes(List named * Converts named expressions to {@link ReferenceAttribute}s, preserving {@link NameId}s for attributes whose name * matches one in {@code existingOutput}. Genuinely new attributes get fresh NameIds. *

- * Exception: a {@link FieldAttribute} backed by an {@link InvalidMappedField} (ambiguous type across indices) is instead + * Exception: a {@link FieldAttribute} backed by a {@link TypeConflictField} (ambiguous type across indices) is instead * converted to an {@link UnsupportedAttribute} via * {@link FieldAttribute#flagTypeConflicts()}, so the analyzer can surface a clear user-facing error. */ @@ -60,7 +60,7 @@ public static List toReferenceAttributesPreservingIds( Attribute existing = existingByName.get(exp.name()); NameId id = existing != null ? existing.id() : new NameId(); Attribute refAttr = switch (exp) { - case FieldAttribute fa when fa.field() instanceof InvalidMappedField -> fa.flagTypeConflicts(); + case FieldAttribute fa when fa.field() instanceof TypeConflictField -> fa.flagTypeConflicts(); case ReferenceAttribute ra -> ra.withId(id); default -> new ReferenceAttribute(exp.source(), null, exp.name(), exp.dataType(), exp.nullable(), id, exp.synthetic()); }; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java index f027cda423cba..f9f53d8633886 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java @@ -16,7 +16,7 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; -import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; +import org.elasticsearch.xpack.esql.core.type.TypeConflictField; import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; @@ -243,12 +243,12 @@ public FieldName fieldName() { } /** - * If the underlying field is an {@link InvalidMappedField} (ambiguous type across indices), + * If the underlying field is a {@link TypeConflictField} (ambiguous type across indices), * converts this attribute into an {@link UnsupportedAttribute} with a descriptive error message * so the analyzer can surface a clear user-facing error. */ public Attribute flagTypeConflicts() { - if (field instanceof InvalidMappedField imf) { + if (field instanceof TypeConflictField imf) { // Field has conflicting types across indices — build a user-facing error message. String unresolvedMessage = "Cannot use field [" + name() + "] due to ambiguities being " + imf.errorMessage(); List types = imf.getTypesToIndices().keySet().stream().toList(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java index ea0988c5e0f89..78d8758879f48 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java @@ -9,7 +9,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression.TypeResolution; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; -import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; +import org.elasticsearch.xpack.esql.core.type.TypeConflictField; import java.util.Locale; import java.util.StringJoiner; @@ -250,7 +250,7 @@ public static TypeResolution isType( // TODO: Shouldn't we perform widening of small numerical types here? if (allowUnionTypes && e instanceof FieldAttribute fa - && fa.field() instanceof InvalidMappedField imf + && fa.field() instanceof TypeConflictField imf && imf.types().stream().allMatch(predicate)) { return TypeResolution.TYPE_RESOLVED; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/CompactInvalidMappedField.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/CompactInvalidMappedField.java new file mode 100644 index 0000000000000..3b3253ad3559f --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/CompactInvalidMappedField.java @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.core.type; + +import org.checkerframework.checker.nullness.qual.NonNull; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException; + +import java.io.IOException; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; + +/** + * Memory-frugal counterpart to {@link InvalidMappedField}: stores at most {@value #MAX_INDICES_PER_TYPE} concrete index names per source + * type instead of the full per-type index list. + */ +public final class CompactInvalidMappedField extends EsField implements TypeConflictField { + private static final int MAX_INDICES_PER_TYPE = 3; + + private final String errorMessage; + private final Map> typesToIndices; + private final boolean isPotentiallyUnmapped; + + public CompactInvalidMappedField(String name, Map> typesToIndices) { + this(name, makeErrorMessage(typesToIndices, false), truncate(typesToIndices), false); + } + + public static CompactInvalidMappedField potentiallyUnmapped(String name, Map> typesToIndices) { + return new CompactInvalidMappedField(name, makeErrorMessage(typesToIndices, true), truncate(typesToIndices), true); + } + + private CompactInvalidMappedField( + String name, + String errorMessage, + Map> typesToIndices, + boolean isPotentiallyUnmapped + ) { + super(name, DataType.UNSUPPORTED, new TreeMap<>(), false, TimeSeriesFieldType.UNKNOWN); + this.errorMessage = errorMessage; + this.typesToIndices = typesToIndices; + this.isPotentiallyUnmapped = isPotentiallyUnmapped; + } + + @Override + public void writeContent(StreamOutput out) throws IOException { + throw new UnsupportedOperationException("CompactInvalidMappedField shouldn't be transported"); + } + + @Override + public String getWriteableName(TransportVersion transportVersion) { + return "InvalidMappedField"; + } + + @Override + public String errorMessage() { + return errorMessage; + } + + @Override + public Map> getTypesToIndices() { + Map> result = new TreeMap<>(); + typesToIndices.forEach((k, v) -> result.put(k.typeName(), v)); + return result; + } + + @Override + public boolean isPotentiallyUnmapped() { + return isPotentiallyUnmapped; + } + + @Override + public Set types() { + return typesToIndices.keySet(); + } + + @Override + public EsField getExactField() { + throw new QlIllegalArgumentException("Field [" + getName() + "] is invalid, cannot access it"); + } + + @Override + public Exact getExactInfo() { + return new Exact(false, "Field [" + getName() + "] is invalid, cannot access it"); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), errorMessage); + } + + @Override + public boolean equals(Object obj) { + if (super.equals(obj) == false) { + return false; + } + CompactInvalidMappedField other = (CompactInvalidMappedField) obj; + return Objects.equals(errorMessage, other.errorMessage); + } + + /** Cap each per-type index set at {@value #MAX_INDICES_PER_TYPE} entries. */ + private static Map> truncate(Map> typesToIndices) { + Map> result = new TreeMap<>(); + for (Map.Entry> entry : typesToIndices.entrySet()) { + Set indices = entry.getValue(); + result.put(entry.getKey(), indices.size() <= MAX_INDICES_PER_TYPE ? Set.copyOf(indices) : truncate(indices)); + } + return result; + } + + private static @NonNull Set truncate(Set indices) { + Set truncated = new LinkedHashSet<>(MAX_INDICES_PER_TYPE + 1); + indices.stream().sorted().limit(MAX_INDICES_PER_TYPE).forEach(truncated::add); + truncated.add("..."); + return Collections.unmodifiableSet(truncated); + } + + /** + * Adapter onto {@link TypeConflictField#makeErrorMessage(Map, boolean)} since that one is shared with {@link InvalidMappedField} and + * therefore takes string keys. The string-keyed view is built ad-hoc here, used to render the message, and discarded. + */ + private static String makeErrorMessage(Map> typesToIndices, boolean includeInsistKeyword) { + Map> stringKeyed = new TreeMap<>(); + typesToIndices.forEach((k, v) -> stringKeyed.put(k.typeName(), v)); + return TypeConflictField.makeErrorMessage(stringKeyed, includeInsistKeyword); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/CompactMultiTypeEsField.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/CompactMultiTypeEsField.java new file mode 100644 index 0000000000000..49466727a2002 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/CompactMultiTypeEsField.java @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.core.type; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.Strings; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; + +import java.io.IOException; +import java.util.Collection; +import java.util.Map; +import java.util.Objects; + +/** + * Memory-efficient variant of {@link MultiTypeEsField} that stores the per-source-type conversion + * expressions directly, rather than expanding them to one entry per index. + */ +public final class CompactMultiTypeEsField extends EsField implements UnionTypeEsField { + public static final TransportVersion CompactMultiTypeEsField = TransportVersion.fromName("compact_multi_type_es_field"); + + // TODO these Expressions should be an AbstractConvertFunction. + private final Map typeToConversionExpressions; + + /** + * If this is not {@code null}, then this expression should be used to convert the field value in case the field is not mapped in an + * index from {@link DataType#KEYWORD} to the target type. + */ + @Nullable + private final Expression unmappedConversionExpression; + + public CompactMultiTypeEsField( + String name, + DataType dataType, + boolean aggregatable, + Map typeToConversionExpressions, + TimeSeriesFieldType timeSeriesFieldType, + @Nullable Expression unmappedConversionExpression + ) { + super(name, dataType, Map.of(), aggregatable, timeSeriesFieldType); + this.typeToConversionExpressions = typeToConversionExpressions; + this.unmappedConversionExpression = unmappedConversionExpression; + } + + CompactMultiTypeEsField(StreamInput in) throws IOException { + this( + ((PlanStreamInput) in).readCachedString(), + DataType.readFrom(in), + in.readBoolean(), + in.readImmutableMap(DataType::readFrom, i -> i.readNamedWriteable(Expression.class)), + readTimeSeriesFieldType(in), + in.readOptionalNamedWriteable(Expression.class) + ); + } + + @Override + public void writeContent(StreamOutput out) throws IOException { + ((PlanStreamOutput) out).writeCachedString(getName()); + getDataType().writeTo(out); + out.writeBoolean(isAggregatable()); + out.writeMap(typeToConversionExpressions, (o, k) -> k.writeTo(o), StreamOutput::writeNamedWriteable); + writeTimeSeriesFieldType(out); + out.writeOptionalNamedWriteable(unmappedConversionExpression); + } + + @Override + public String getWriteableName(TransportVersion transportVersion) { + return getNodeStringName(); + } + + @Override + public String getNodeStringName() { + return "CompactMultiTypeEsField"; + } + + public Map getTypeToConversionExpressions() { + return typeToConversionExpressions; + } + + @Override + public Collection getConversionExpressions() { + return typeToConversionExpressions.values(); + } + + @Override + public EsField rewrapWithCast(Expression convertExpression) { + return new CompactMultiTypeEsField( + getName(), + convertExpression.dataType(), + isAggregatable(), + UnionTypeEsField.replaceChildrenWithExpressionField(typeToConversionExpressions, convertExpression), + getTimeSeriesFieldType(), + unmappedConversionExpression + ); + } + + /** + * Returns the conversion expression to apply for the given source {@link DataType}, or {@code null} + * if no conversion is registered for that type. + */ + public @Nullable Expression getConversionExpressionForType(DataType type) { + return typeToConversionExpressions.get(type); + } + + @Override + public @Nullable Expression getUnmappedConversionExpression() { + return unmappedConversionExpression; + } + + public static CompactMultiTypeEsField resolveFrom( + TypeConflictField imf, + Map typesToConversionExpressions, + @Nullable Expression unmappedConversionExpression + ) { + UnionTypeEsField.Resolution resolution = UnionTypeEsField.resolve(imf, typesToConversionExpressions); + DataType resolvedDataType = resolution.resolvedDataType() == DataType.UNSUPPORTED && unmappedConversionExpression != null + ? unmappedConversionExpression.dataType() + : resolution.resolvedDataType(); + return new CompactMultiTypeEsField( + imf.getName(), + resolvedDataType, + false, + resolution.typeToExpr(), + imf.getTimeSeriesFieldType(), + unmappedConversionExpression + ); + } + + @Override + public boolean equals(Object obj) { + if (super.equals(obj) == false) { + return false; + } + if (obj instanceof CompactMultiTypeEsField other) { + return typeToConversionExpressions.equals(other.typeToConversionExpressions) + && Objects.equals(unmappedConversionExpression, other.unmappedConversionExpression); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), typeToConversionExpressions, unmappedConversionExpression); + } + + @Override + public String toString() { + return Strings.format("%s (%s, %s)", super.toString(), typeToConversionExpressions, unmappedConversionExpression); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java index 9c47b029431af..be9208ef4f73d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java @@ -107,6 +107,7 @@ public static TimeSeriesFieldType fromIndexFieldCapabilities(IndexFieldCapabilit Map.entry("KeywordEsField", KeywordEsField::new), Map.entry("MissingEsField", MissingEsField::new), Map.entry("MultiTypeEsField", MultiTypeEsField::new), + Map.entry("CompactMultiTypeEsField", CompactMultiTypeEsField::new), Map.entry("PotentiallyUnmappedKeywordEsField", PotentiallyUnmappedKeywordEsField::new), Map.entry("TextEsField", TextEsField::new), Map.entry("UnsupportedEsField", UnsupportedEsField::new) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java index e110b0006922b..74355913beb88 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java @@ -22,6 +22,9 @@ import java.util.stream.Collectors; /** + *

+ * N.B.: This class exists only as a backward-compatible version of {@link CompactInvalidMappedField}. + *

* Representation of field mapped differently across indices; or being potentially unmapped in some, in which case it is treated as * {@link DataType#KEYWORD} in the indices where it is unmapped. * Used during mapping discovery only. @@ -29,7 +32,7 @@ * not required through the cluster, only surviving as long as the Analyser phase of query planning. * It is used specifically for the 'union types' and 'unmapped fields' feature in ES|QL. */ -public class InvalidMappedField extends EsField { +public final class InvalidMappedField extends EsField implements TypeConflictField { private final String errorMessage; private final Map> typesToIndices; @@ -44,7 +47,14 @@ public InvalidMappedField(String name, String errorMessage) { } public InvalidMappedField(String name, Map> typesToIndices) { - this(name, makeErrorMessage(typesToIndices, false), new TreeMap<>(), typesToIndices, false, TimeSeriesFieldType.UNKNOWN); + this( + name, + TypeConflictField.makeErrorMessage(typesToIndices, false), + new TreeMap<>(), + typesToIndices, + false, + TimeSeriesFieldType.UNKNOWN + ); } /** @@ -55,7 +65,7 @@ public InvalidMappedField(String name, Map> typesToIndices) public static InvalidMappedField potentiallyUnmapped(String name, Map> typesToIndices) { return new InvalidMappedField( name, - makeErrorMessage(typesToIndices, true), + TypeConflictField.makeErrorMessage(typesToIndices, true), new TreeMap<>(), typesToIndices, true, @@ -77,7 +87,7 @@ private InvalidMappedField( this.isPotentiallyUnmapped = isPotentiallyUnmapped; } - protected InvalidMappedField(StreamInput in) throws IOException { + InvalidMappedField(StreamInput in) throws IOException { this( ((PlanStreamInput) in).readCachedString(), in.readString(), @@ -88,10 +98,6 @@ protected InvalidMappedField(StreamInput in) throws IOException { ); } - public Set types() { - return typesToIndices.keySet().stream().map(DataType::fromTypeName).collect(Collectors.toSet()); - } - @Override public void writeContent(StreamOutput out) throws IOException { ((PlanStreamOutput) out).writeCachedString(getName()); @@ -104,6 +110,7 @@ public String getWriteableName(TransportVersion transportVersion) { return "InvalidMappedField"; } + @Override public String errorMessage() { return errorMessage; } @@ -134,47 +141,18 @@ public Exact getExactInfo() { return new Exact(false, "Field [" + getName() + "] is invalid, cannot access it"); } + @Override public Map> getTypesToIndices() { return typesToIndices; } + @Override public boolean isPotentiallyUnmapped() { return isPotentiallyUnmapped; } - private static String makeErrorMessage(Map> typesToIndices, boolean includeInsistKeyword) { - StringBuilder errorMessage = new StringBuilder(); - var isInsistKeywordOnlyKeyword = includeInsistKeyword && typesToIndices.containsKey(DataType.KEYWORD.typeName()) == false; - errorMessage.append("mapped as ["); - errorMessage.append(typesToIndices.size() + (isInsistKeywordOnlyKeyword ? 1 : 0)); - errorMessage.append("] incompatible types: "); - boolean first = true; - if (isInsistKeywordOnlyKeyword) { - first = false; - errorMessage.append("[keyword] due to loading from _source"); - } - for (Map.Entry> e : typesToIndices.entrySet()) { - if (first) { - first = false; - } else { - errorMessage.append(", "); - } - errorMessage.append("["); - errorMessage.append(e.getKey()); - errorMessage.append("] "); - if (e.getKey().equals(DataType.KEYWORD.typeName()) && includeInsistKeyword) { - errorMessage.append("due to loading from _source and in "); - } else { - errorMessage.append("in "); - } - if (e.getValue().size() <= 3) { - errorMessage.append(e.getValue()); - } else { - errorMessage.append(e.getValue().stream().sorted().limit(3).collect(Collectors.toList())); - errorMessage.append(" and [" + (e.getValue().size() - 3) + "] other "); - errorMessage.append(e.getValue().size() == 4 ? "index" : "indices"); - } - } - return errorMessage.toString(); + @Override + public Set types() { + return getTypesToIndices().keySet().stream().map(DataType::fromTypeName).collect(Collectors.toSet()); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java index d825d13c32763..07e8870d5e113 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java @@ -17,12 +17,15 @@ import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; import java.io.IOException; +import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Objects; -import java.util.Set; /** + *

+ * N.B.: This class exists only as a backward-compatible version of {@link CompactInvalidMappedField}. + *

* During IndexResolution it can occur that the same field is mapped to different types in different indices. * An {@link InvalidMappedField} holds that information and allows for later resolution of the field * to a single type in {@code ResolveUnionTypes}. @@ -31,7 +34,7 @@ * this class instead of the {@link InvalidMappedField}. * This class is sent to the data nodes to inform them that they have to convert the type directly during field extraction. */ -public class MultiTypeEsField extends EsField { +public final class MultiTypeEsField extends EsField implements UnionTypeEsField { private static final TransportVersion POTENTIALLY_UNMAPPED_EXPRESSION = TransportVersion.fromName( "esql_potentially_unmapped_expression" ); @@ -94,10 +97,32 @@ public String getNodeStringName() { return potentiallyUnmappedExpression; } + @Override + public @Nullable Expression getUnmappedConversionExpression() { + return potentiallyUnmappedExpression; + } + public Map getIndexToConversionExpressions() { return indexToConversionExpressions; } + @Override + public Collection getConversionExpressions() { + return indexToConversionExpressions.values(); + } + + @Override + public EsField rewrapWithCast(Expression convertExpression) { + return new MultiTypeEsField( + getName(), + convertExpression.dataType(), + isAggregatable(), + UnionTypeEsField.replaceChildrenWithExpressionField(indexToConversionExpressions, convertExpression), + getTimeSeriesFieldType(), + potentiallyUnmappedExpression + ); + } + public @Nullable Expression getConversionExpressionForIndex(String indexName) { return indexToConversionExpressions.get(indexName); } @@ -114,30 +139,23 @@ public MultiTypeEsField withPotentiallyUnmappedExpression(@Nullable Expression p } public static MultiTypeEsField resolveFrom( - InvalidMappedField invalidMappedField, + TypeConflictField typeConflictedField, Map typesToConversionExpressions ) { - Map> typesToIndices = invalidMappedField.getTypesToIndices(); - DataType resolvedDataType = DataType.UNSUPPORTED; + UnionTypeEsField.Resolution resolution = UnionTypeEsField.resolve(typeConflictedField, typesToConversionExpressions); Map indexToConversionExpressions = new HashMap<>(); - for (String typeName : typesToIndices.keySet()) { - Set indices = typesToIndices.get(typeName); - Expression convertExpr = typesToConversionExpressions.get(typeName); - if (resolvedDataType == DataType.UNSUPPORTED) { - resolvedDataType = convertExpr.dataType(); - } else if (resolvedDataType != convertExpr.dataType()) { - throw new IllegalArgumentException("Resolved data type mismatch: " + resolvedDataType + " != " + convertExpr.dataType()); - } + typeConflictedField.getTypesToIndices().forEach((typeName, indices) -> { + Expression convertExpr = resolution.typeToExpr().get(DataType.fromTypeName(typeName)); for (String indexName : indices) { indexToConversionExpressions.put(indexName, convertExpr); } - } + }); return new MultiTypeEsField( - invalidMappedField.getName(), - resolvedDataType, + typeConflictedField.getName(), + resolution.resolvedDataType(), false, indexToConversionExpressions, - invalidMappedField.getTimeSeriesFieldType(), + typeConflictedField.getTimeSeriesFieldType(), null ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/TypeConflictField.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/TypeConflictField.java new file mode 100644 index 0000000000000..3df36cfd98116 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/TypeConflictField.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.core.type; + +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public sealed interface TypeConflictField permits InvalidMappedField, CompactInvalidMappedField { + String getName(); + + Map getProperties(); + + boolean isAggregatable(); + + EsField.TimeSeriesFieldType getTimeSeriesFieldType(); + + /** + * Pre-rendered, user-facing error message describing the conflict. Built from the full input map at construction time so it + * survives the index-list truncation done by {@link CompactInvalidMappedField}. + */ + String errorMessage(); + + /** + * Per-source-type indices in which the field appears with that type. Note that {@link CompactInvalidMappedField} caps each set + * and may include the {@code "..."} sentinel; callers that need a complete index list should use {@link InvalidMappedField} + * instead. + */ + Map> getTypesToIndices(); + + /** Whether the field is unmapped in at least one index, in which case it's treated as {@link DataType#KEYWORD} where it is unmapped. */ + boolean isPotentiallyUnmapped(); + + /** Source data types observed for this field across all indices. */ + Set types(); + + /** + * Build the user-facing error message for a per-type-to-indices map. Shared between both implementations so they stay in sync. + */ + static String makeErrorMessage(Map> typesToIndices, boolean includeInsistKeyword) { + StringBuilder errorMessage = new StringBuilder(); + var isInsistKeywordOnlyKeyword = includeInsistKeyword && typesToIndices.containsKey(DataType.KEYWORD.typeName()) == false; + errorMessage.append("mapped as ["); + errorMessage.append(typesToIndices.size() + (isInsistKeywordOnlyKeyword ? 1 : 0)); + errorMessage.append("] incompatible types: "); + boolean first = true; + if (isInsistKeywordOnlyKeyword) { + first = false; + errorMessage.append("[keyword] due to loading from _source"); + } + for (Map.Entry> e : typesToIndices.entrySet()) { + if (first) { + first = false; + } else { + errorMessage.append(", "); + } + errorMessage.append("["); + errorMessage.append(e.getKey()); + errorMessage.append("] "); + if (e.getKey().equals(DataType.KEYWORD.typeName()) && includeInsistKeyword) { + errorMessage.append("due to loading from _source and in "); + } else { + errorMessage.append("in "); + } + if (e.getValue().size() <= 3) { + errorMessage.append(e.getValue()); + } else { + errorMessage.append(e.getValue().stream().sorted().limit(3).collect(Collectors.toList())); + errorMessage.append(" and [" + (e.getValue().size() - 3) + "] other "); + errorMessage.append(e.getValue().size() == 4 ? "index" : "indices"); + } + } + return errorMessage.toString(); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/UnionTypeEsField.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/UnionTypeEsField.java new file mode 100644 index 0000000000000..25f7616f6a307 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/UnionTypeEsField.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.core.type; + +import org.elasticsearch.core.Nullable; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.util.CollectionUtils; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Common interface implemented by both {@link MultiTypeEsField} (legacy, keyed by index name) and + * {@link CompactMultiTypeEsField} (newer, keyed by source data type) so that callers that only care about + * the existence of a per-(index|type) conversion or about the unmapped-side conversion can treat the + * two implementations uniformly. + */ +public sealed interface UnionTypeEsField permits MultiTypeEsField, CompactMultiTypeEsField { + /** + * Conversion expression to apply when the field is unmapped in the index, treating it as {@link DataType#KEYWORD}, or {@code null} + * if there is no such conversion (i.e., unmapped indices should produce {@code null}). + */ + @Nullable + Expression getUnmappedConversionExpression(); + + Collection getConversionExpressions(); + + /** + * Wraps an existing union-type field's per-(index|type) conversions with another conversion expression on top, so the + * composite expression first does the original cast then the additional cast. + */ + EsField rewrapWithCast(Expression convertExpression); + + static Map replaceChildrenWithExpressionField(Map map, Expression expression) { + return CollectionUtils.mapValues(map, e -> expression.replaceChildren(List.of(((AbstractConvertFunction) e).field()))); + } + + static Resolution resolve(TypeConflictField field, Map typesToConversionExpressions) { + DataType resolvedDataType = DataType.UNSUPPORTED; + Map typeToExpr = new HashMap<>(); + for (String typeName : field.getTypesToIndices().keySet()) { + Expression convertExpr = typesToConversionExpressions.get(typeName); + if (resolvedDataType == DataType.UNSUPPORTED) { + resolvedDataType = convertExpr.dataType(); + } else if (resolvedDataType != convertExpr.dataType()) { + throw new IllegalArgumentException("Resolved data type mismatch: " + resolvedDataType + " != " + convertExpr.dataType()); + } + typeToExpr.put(DataType.fromTypeName(typeName), convertExpr); + } + return new Resolution(resolvedDataType, typeToExpr); + } + + record Resolution(DataType resolvedDataType, Map typeToExpr) {} +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/util/CollectionUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/util/CollectionUtils.java index e8dfa260fd803..03d9d5524546b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/util/CollectionUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/util/CollectionUtils.java @@ -9,8 +9,11 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.function.Function; import static java.util.Collections.emptyList; @@ -90,4 +93,12 @@ public static List prependToCopy(T element, Collection collection) { } return List.of(result); } + + public static Map mapValues(Map map, Function f) { + var res = new HashMap(); + for (var entry : map.entrySet()) { + res.put(entry.getKey(), f.apply(entry.getValue())); + } + return res; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java index ced68bff788e8..10f474f4f26ae 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java @@ -36,7 +36,7 @@ import org.elasticsearch.xpack.esql.core.tree.Node; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; +import org.elasticsearch.xpack.esql.core.type.UnionTypeEsField; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; @@ -516,13 +516,13 @@ private IndexedByShardId toShardConfigs(IndexedByShardId new ShardConfig(sc.toQuery(evaluatorQueryBuilder()), sc.searcher())); } - // TODO: this should likely be replaced by calls to FieldAttribute#fieldName; the MultiTypeEsField case looks - // wrong if `fieldAttribute` is a subfield, e.g. `parent.child` - multiTypeEsField#getName will just return `child`. + // TODO: this should likely be replaced by calls to FieldAttribute#fieldName; the UnionTypeEsField case looks + // wrong if `fieldAttribute` is a subfield, e.g. `parent.child` - EsField#getName will just return `child`. protected String getNameFromFieldAttribute(FieldAttribute fieldAttribute) { String fieldName = fieldAttribute.name(); - if (fieldAttribute.field() instanceof MultiTypeEsField multiTypeEsField) { - // If we have multiple field types, we allow the query to be done, but getting the underlying field name - fieldName = multiTypeEsField.getName(); + if (fieldAttribute.field() instanceof UnionTypeEsField) { + // If we have multiple field types, we allow the query to be done, but get the underlying field name + fieldName = fieldAttribute.field().getName(); } return fieldName; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDouble.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDouble.java index 96ee4d9030b95..024a99abfd7e4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDouble.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDouble.java @@ -29,7 +29,7 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; +import org.elasticsearch.xpack.esql.core.type.UnionTypeEsField; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.blockloader.BlockLoaderExpression; @@ -201,7 +201,7 @@ public Set supportedTypes() { public PushedBlockLoaderExpression tryPushToFieldLoading(SearchStats stats) { if (field() instanceof FieldAttribute f && f.dataType() == AGGREGATE_METRIC_DOUBLE - && (f.field() instanceof MultiTypeEsField) == false) { + && (f.field() instanceof UnionTypeEsField) == false) { var folded = subfieldIndex.fold(FoldContext.small()); if (folded == null) { throw new IllegalArgumentException("Subfield Index was null"); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java index 351139095cd46..aa2471c9bd97e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java @@ -9,7 +9,6 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.xpack.esql.core.type.EsField; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -19,23 +18,12 @@ public record EsIndex( Map mapping, // keyed by field names Map indexNameWithModes, Map> originalIndices, // keyed by cluster alias - Map> concreteIndices, // keyed by cluster alias - Map> fieldToUnmappedIndices // keyed by field name; Set are concrete index names. + Map> concreteIndices // keyed by cluster alias ) { public EsIndex { assert name != null; assert mapping != null; - assert fieldToUnmappedIndices != null; - assert fieldToUnmappedIndices.values().stream().noneMatch(Set::isEmpty); - } - - public boolean isPartiallyUnmappedField(String fieldName) { - return fieldToUnmappedIndices.containsKey(fieldName); - } - - public Set getUnmappedIndices(String fieldName) { - return fieldToUnmappedIndices.getOrDefault(fieldName, Collections.emptySet()); } public Set concreteQualifiedIndices() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java index 752e95bc1e20f..4576931789d87 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java @@ -40,7 +40,7 @@ public static IndexResolution valid(EsIndex index) { } public static IndexResolution empty(String indexPattern) { - return valid(new EsIndex(indexPattern, Map.of(), Map.of(), Map.of(), Map.of(), Map.of())); + return valid(new EsIndex(indexPattern, Map.of(), Map.of(), Map.of(), Map.of())); } public static IndexResolution invalid(String invalid) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java index cf0f5f9cc053e..1976d22f832d2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java @@ -33,7 +33,7 @@ import org.elasticsearch.xpack.esql.optimizer.rules.logical.PropagateEvalFoldables; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PropagateInlineEvals; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PropagateNullable; -import org.elasticsearch.xpack.esql.optimizer.rules.logical.PropgateUnmappedFields; +import org.elasticsearch.xpack.esql.optimizer.rules.logical.PropagateUnmappedFields; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PruneColumns; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PruneEmptyAggregates; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PruneEmptyForkBranches; @@ -269,7 +269,7 @@ protected static Batch cleanup() { new ReplaceLimitAndSortAsTopN(), new HoistRemoteEnrichTopN(), new ReplaceRowAsLocalRelation(), - new PropgateUnmappedFields(), + new PropagateUnmappedFields(), new CombineLimitTopN(), new ReorderLimitProjectAndOrderBy() ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateUnmappedFields.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateUnmappedFields.java new file mode 100644 index 0000000000000..b96b777f0a19c --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateUnmappedFields.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules.logical; + +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.AttributeSet; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; +import org.elasticsearch.xpack.esql.expression.NamedExpressions; +import org.elasticsearch.xpack.esql.plan.logical.EsRelation; +import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.rule.Rule; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Merges unmapped fields into the output of the ES relation. This marking is necessary for the block loaders to force loading from _source + * if the field is unmapped. + * + * N.B. This is only used for INSIST keyword, so when INSIST is sunset, we can get rid of this rule! + */ +public class PropagateUnmappedFields extends Rule { + @Override + public LogicalPlan apply(LogicalPlan logicalPlan) { + if (logicalPlan instanceof EsRelation) { + return logicalPlan; + } + var unmappedFieldsBuilder = AttributeSet.builder(); + logicalPlan.forEachExpressionDown(FieldAttribute.class, fa -> { + if (fa.field() instanceof PotentiallyUnmappedKeywordEsField) { + unmappedFieldsBuilder.add(fa); + } + }); + var unmappedFields = unmappedFieldsBuilder.build(); + return unmappedFields.isEmpty() ? logicalPlan : logicalPlan.transformUp(EsRelation.class, er -> mergeMissing(er, unmappedFields)); + } + + private static EsRelation mergeMissing(EsRelation er, AttributeSet unmappedFields) { + Set existingPuks = er.output() + .stream() + .flatMap( + attr -> attr instanceof FieldAttribute fa && fa.field() instanceof PotentiallyUnmappedKeywordEsField + ? Stream.of(fa.fieldName().string()) + : Stream.empty() + ) + .collect(Collectors.toUnmodifiableSet()); + // Only propagate PUK fields that are not already in the relation's output, so we preserve the existing order. + // Partially-mapped keyword fields are already wrapped as PUKs by the index resolver; this rule just merges in + // PUKs introduced elsewhere (e.g., by INSIST on a field that is not in the index). + List missing = unmappedFields.stream() + .flatMap( + attr -> attr instanceof FieldAttribute fa && existingPuks.contains(fa.fieldName().string()) == false + ? Stream.of(attr) + : Stream.empty() + ) + .toList(); + return missing.isEmpty() ? er : er.withAttributes(NamedExpressions.mergeOutputAttributes(missing, er.output())); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropgateUnmappedFields.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropgateUnmappedFields.java deleted file mode 100644 index 23f2110d628cc..0000000000000 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropgateUnmappedFields.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.optimizer.rules.logical; - -import org.elasticsearch.xpack.esql.core.expression.AttributeSet; -import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; -import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; -import org.elasticsearch.xpack.esql.expression.NamedExpressions; -import org.elasticsearch.xpack.esql.plan.logical.EsRelation; -import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; -import org.elasticsearch.xpack.esql.rule.Rule; - -import java.util.ArrayList; - -/** - * Merges unmapped fields into the output of the ES relation. This marking is necessary for the block loaders to force loading from _source - * if the field is unmapped. - */ -public class PropgateUnmappedFields extends Rule { - @Override - public LogicalPlan apply(LogicalPlan logicalPlan) { - if (logicalPlan instanceof EsRelation) { - return logicalPlan; - } - var unmappedFieldsBuilder = AttributeSet.builder(); - logicalPlan.forEachExpressionDown(FieldAttribute.class, fa -> { - if (fa.field() instanceof PotentiallyUnmappedKeywordEsField) { - unmappedFieldsBuilder.add(fa); - } - }); - var unmappedFields = unmappedFieldsBuilder.build(); - return unmappedFields.isEmpty() - ? logicalPlan - : logicalPlan.transformUp( - EsRelation.class, - er -> hasPotentiallyUnmappedKeywordEsField(er) - ? er - : er.withAttributes(NamedExpressions.mergeOutputAttributes(new ArrayList<>(unmappedFields), er.output())) - ); - } - - // Checks if the EsRelation already has a PotentiallyUnmappedKeywordEsField. If true SET load_unmapped="load" is applied. - // This is used to practically disable the rule, since it changes the output order (mergeOutputAttributes()). - private static boolean hasPotentiallyUnmappedKeywordEsField(EsRelation er) { - for (var attr : er.output()) { - if (attr instanceof FieldAttribute fa && fa.field() instanceof PotentiallyUnmappedKeywordEsField) { - return true; - } - } - return false; - } -} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceDateTruncBucketWithRoundTo.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceDateTruncBucketWithRoundTo.java index 3fa7c1b57a9a7..f1ada34dfd382 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceDateTruncBucketWithRoundTo.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceDateTruncBucketWithRoundTo.java @@ -19,7 +19,7 @@ import org.elasticsearch.xpack.esql.core.expression.function.Function; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; +import org.elasticsearch.xpack.esql.core.type.UnionTypeEsField; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket; import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc; @@ -96,7 +96,7 @@ private RoundTo maybeSubstituteWithRoundTo( Eval eval, TriFunction roundingFunction ) { - if (field instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField == false && isDateTime(fa.dataType())) { + if (field instanceof FieldAttribute fa && fa.field() instanceof UnionTypeEsField == false && isDateTime(fa.dataType())) { DataType fieldType = fa.dataType(); FieldAttribute.FieldName fieldName = fa.fieldName(); // Extract min/max from SearchStats diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceFieldWithConstantOrNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceFieldWithConstantOrNull.java index 67a5f2965a888..e92f6d600c6bf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceFieldWithConstantOrNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceFieldWithConstantOrNull.java @@ -17,8 +17,8 @@ import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.TimeSeriesMetadataAttribute; import org.elasticsearch.xpack.esql.core.type.MissingEsField; -import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; +import org.elasticsearch.xpack.esql.core.type.UnionTypeEsField; import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.optimizer.LocalLogicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.rules.RuleUtils; @@ -99,7 +99,7 @@ else if (esRelation.indexMode() == IndexMode.STANDARD) { private static boolean isPotentiallyUnmapped(FieldAttribute f) { return f.field() instanceof PotentiallyUnmappedKeywordEsField - || (f.field() instanceof MultiTypeEsField mtf && mtf.getPotentiallyUnmappedExpression() != null); + || (f.field() instanceof UnionTypeEsField utf && utf.getUnmappedConversionExpression() != null); } private LogicalPlan replaceWithNullOrConstant( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 85bedb48fcf7b..58906817d83d6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -80,11 +80,13 @@ import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.TemporalityAttribute; import org.elasticsearch.xpack.esql.core.expression.TimeSeriesMetadataAttribute; +import org.elasticsearch.xpack.esql.core.type.CompactMultiTypeEsField; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.FunctionEsField; import org.elasticsearch.xpack.esql.core.type.KeywordEsField; import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; +import org.elasticsearch.xpack.esql.core.type.UnionTypeEsField; import org.elasticsearch.xpack.esql.expression.function.BlockLoaderWarnings; import org.elasticsearch.xpack.esql.expression.function.blockloader.BlockLoaderExpression; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; @@ -99,6 +101,7 @@ import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner.PhysicalOperation; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import org.elasticsearch.xpack.esql.stats.SearchStats; +import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry; import java.io.IOException; import java.util.ArrayList; @@ -249,7 +252,7 @@ private ValuesSourceReaderOperator.LoaderAndConverter blockLoaderAndConverter( functionConfig = functionEsField.functionConfig(); } boolean isUnsupported = attr.dataType() == DataType.UNSUPPORTED; - MultiTypeEsField unionTypes = findUnionTypes(attr); + UnionTypeEsField unionTypes = findUnionTypes(attr); if (unionTypes == null) { BlockLoader blockLoader = shardContext.blockLoader( fieldName, @@ -262,11 +265,25 @@ private ValuesSourceReaderOperator.LoaderAndConverter blockLoaderAndConverter( ); return ValuesSourceReaderOperator.load(blockLoader); } - // Use the fully qualified name `cluster:index-name` because multiple types are resolved on coordinator with the cluster prefix - String indexName = shardContext.ctx.getFullyQualifiedIndex().getName(); - Expression conversion = unionTypes.getConversionExpressionForIndex(indexName); + Expression conversion = switch (unionTypes) { + case CompactMultiTypeEsField compact -> { + MappedFieldType mft = shardContext.fieldType(fieldName); + // Match what field_caps reports on the coordinator: family type (e.g., constant_keyword -> keyword) rather than the + // concrete mapper type, so the lookup key here aligns with how typeToConversionExpressions was keyed upstream. + yield mft == null + ? null + : compact.getConversionExpressionForType( + EsqlDataTypeRegistry.INSTANCE.fromEs(mft.familyTypeName(), mft.getMetricType()) + ); + } + case MultiTypeEsField legacy -> { + // Use the fully qualified name `cluster:index-name` because multiple types are resolved on coordinator with cluster prefix + String indexName = shardContext.ctx.getFullyQualifiedIndex().getName(); + yield legacy.getConversionExpressionForIndex(indexName); + } + }; if (conversion == null) { - Expression potentiallyUnmapped = unionTypes.getPotentiallyUnmappedExpression(); + Expression potentiallyUnmapped = unionTypes.getUnmappedConversionExpression(); if (!(potentiallyUnmapped instanceof AbstractConvertFunction convert)) { return ValuesSourceReaderOperator.LOAD_CONSTANT_NULLS; } @@ -394,9 +411,9 @@ static MappedFieldType createUnmappedFieldType(String name, DefaultShardContext } } - private static @Nullable MultiTypeEsField findUnionTypes(Attribute attr) { - if (attr instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField multiTypeEsField) { - return multiTypeEsField; + private static @Nullable UnionTypeEsField findUnionTypes(Attribute attr) { + if (attr instanceof FieldAttribute fa && fa.field() instanceof UnionTypeEsField unionTypeEsField) { + return unionTypeEsField; } return null; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index f045dd00e95c8..a39aebc22bc0b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -1250,7 +1250,6 @@ private IndexResolution checkSingleIndex( lookupIndexResolution.get().mapping(), Map.of(indexName, IndexMode.LOOKUP), Map.of(), - Map.of(), Map.of() ); return IndexResolution.valid(newIndex, newIndex.concreteQualifiedIndices(), lookupIndexResolution.failures()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index 17e053d7327da..8f2d87cb7c006 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -37,8 +37,10 @@ import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; import org.elasticsearch.xpack.esql.core.type.KeywordEsField; +import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.core.type.SupportedVersion; import org.elasticsearch.xpack.esql.core.type.TextEsField; +import org.elasticsearch.xpack.esql.core.type.TypeConflictField; import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.index.IndexResolution; @@ -53,7 +55,6 @@ import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; -import java.util.stream.Collectors; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; @@ -330,15 +331,6 @@ public static IndexResolution mergedMappings( String[] names = fieldsCaps.keySet().toArray(new String[0]); Arrays.sort(names); Map rootFields = new HashMap<>(); - Map> fieldToUnmappedIndices; - Set allIndexNames; - if (trackUnmappedFieldIndices) { - fieldToUnmappedIndices = new HashMap<>(); - allIndexNames = indexResponses.stream().map(FieldCapabilitiesIndexResponse::getIndexName).collect(Collectors.toSet()); - } else { - fieldToUnmappedIndices = Map.of(); - allIndexNames = null; - } for (String name : names) { Map fields = rootFields; String fullName = name; @@ -374,14 +366,11 @@ public static IndexResolution mergedMappings( firstUnsupportedParent.getName(), new HashMap<>() ); - fields.put(name, field); if (trackUnmappedFieldIndices) { - Set unmappedIndices = new TreeSet<>(allIndexNames); - unmappedIndices.removeAll(collectedFieldCaps.fieldToMappedIndices.getOrDefault(fullName, Set.of())); - if (unmappedIndices.isEmpty() == false) { - fieldToUnmappedIndices.put(fullName, unmappedIndices); - } + Set mappedIndices = collectedFieldCaps.fieldToMappedIndices.getOrDefault(fullName, Set.of()); + field = wrapIfPartiallyUnmapped(field, name, fullName, mappedIndices, numberOfIndices); } + fields.put(name, field); } boolean allEmpty = true; @@ -409,8 +398,7 @@ public static IndexResolution mergedMappings( // FieldCapabilitiesResponse#resolvedLocally and FieldCapabilitiesResponse#resolvedRemotely // once all remotes support it (v9.3+) originalIndexExtractor.apply(indexPattern, fieldsInfo.caps), - concreteIndices, - fieldToUnmappedIndices + concreteIndices ); var failures = EsqlCCSUtils.groupFailuresPerCluster(fieldsInfo.caps.getFailures()); return IndexResolution.valid(index, indexNameWithModes.keySet(), failures); @@ -528,6 +516,36 @@ private static EsField createField( return new EsField(name, type, new HashMap<>(), aggregatable, isAlias, timeSeriesFieldType); } + // Visible for testing. + public static EsField wrapIfPartiallyUnmapped( + EsField field, + String name, + String fullName, + Set mappedIndices, + int numberOfIndices + ) { + return field instanceof UnsupportedEsField == false && mappedIndices.size() < numberOfIndices + ? wrapPartiallyUnmappedField(field, name, fullName, mappedIndices) + : field; + } + + // Visible for testing + public static EsField wrapPartiallyUnmappedField(EsField field, String name, String fullName, Set mappedIndices) { + return switch (field.getDataType()) { + // OBJECT fields are containers for subfields, not leaf fields that get queried directly. + // Wrapping them would break downstream code that doesn't expect OBJECT as a data type in InvalidMappedField. + case OBJECT -> field; + // PotentiallyUnmappedKeywordEsField needs the full dotted path for DefaultShardContextForUnmappedField.fieldType(). + case KEYWORD -> new PotentiallyUnmappedKeywordEsField(fullName); + default -> InvalidMappedField.potentiallyUnmapped( + name, + field instanceof TypeConflictField imf + ? imf.getTypesToIndices() + : Map.of(field.getDataType().widenSmallNumeric().typeName(), mappedIndices) + ); + }; + } + private static UnsupportedEsField unsupported(String name, IndexFieldCapabilities fc) { String originalType = fc.metricType() == TimeSeriesParams.MetricType.COUNTER ? "counter" : fc.type(); return new UnsupportedEsField(name, List.of(originalType)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java index fc74838cea8b0..5f517e29a1e96 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java @@ -118,7 +118,6 @@ public static IndexResolution indexWithDateDateNanosUnionType() { Map.of(dateDateNanos, dateDateNanosField, dateDateNanosLong, dateDateNanosLongField), Map.of("index1", IndexMode.STANDARD, "index2", IndexMode.STANDARD, "index3", IndexMode.STANDARD), Map.of(), - Map.of(), Map.of() ); return IndexResolution.valid(index); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index e26ffa7487513..d2f72fc2ac6d9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -49,8 +49,8 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; -import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; +import org.elasticsearch.xpack.esql.core.type.UnionTypeEsField; import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy; import org.elasticsearch.xpack.esql.expression.Order; import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; @@ -4695,7 +4695,6 @@ public void testProjectionForUnionTypeResolution() { Map.of("id", idField, "foo", fooField), // Updated mapping keys Map.of("union_index_1", IndexMode.STANDARD, "union_index_2", IndexMode.STANDARD), Map.of(), - Map.of(), Map.of() ); IndexResolution resolution = IndexResolution.valid(index); @@ -4738,7 +4737,6 @@ public void testExplicitRetainOriginalFieldWithCast() { Map.of("id", idField), Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), Map.of(), - Map.of(), Map.of() ); IndexResolution resolution = IndexResolution.valid(index); @@ -5011,7 +5009,6 @@ public void testImplicitCastingForAggregateMetricDouble() { mapping, Map.of("k8s", IndexMode.TIME_SERIES, "k8s-downsampled", IndexMode.TIME_SERIES), Map.of(), - Map.of(), Map.of() ); var testAnalyzer = analyzer().addIndex(esIndex); @@ -6740,7 +6737,7 @@ private void verifyNameAndTypeAndMultiTypeEsField( } private boolean isMultiTypeEsField(Expression e) { - return e instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField; + return e instanceof FieldAttribute fa && fa.field() instanceof UnionTypeEsField; } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index 10bc31dd6e63a..2b4c9ef3c8fc1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -34,15 +34,16 @@ import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.OrderBy; import org.elasticsearch.xpack.esql.plan.logical.Project; +import org.elasticsearch.xpack.esql.session.IndexResolver; import org.hamcrest.Matcher; import org.hamcrest.Matchers; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; import static java.util.Collections.emptyMap; import static org.elasticsearch.xpack.esql.EsqlTestUtils.analyzer; @@ -55,6 +56,7 @@ import static org.elasticsearch.xpack.esql.analysis.AnalyzerTests.withInlinestatsWarning; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; @@ -862,23 +864,17 @@ public void testPartiallyMappedNonKeywordFieldsMarkedAsPotentiallyUnmapped() { if (excludedTypes.contains(dataType)) { continue; } - // Build a minimal mapping: one keyword field (emp_no stand-in for SORT) and one field of the type under test + // Build a minimal mapping: one keyword field (emp_no stand-in for SORT) and one field of the type under test, + // with the latter wrapped as InvalidMappedField.potentiallyUnmapped (as IndexResolver would do in production). Map mapping = Map.of( "sort_field", new EsField("sort_field", DataType.INTEGER, Map.of(), true, EsField.TimeSeriesFieldType.NONE), "test_field", - new EsField("test_field", dataType, Map.of(), true, EsField.TimeSeriesFieldType.NONE) + InvalidMappedField.potentiallyUnmapped("test_field", Map.of(dataType.widenSmallNumeric().typeName(), Set.of("test1"))) ); var plan = analyzer().addIndex( - new EsIndex( - "test*", - mapping, - Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), - Map.of(), - Map.of(), - Map.of("test_field", Set.of("test2")) // partially unmapped - ) + new EsIndex("test*", mapping, Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), Map.of(), Map.of()) ).statement(setUnmappedLoad(""" FROM test* | SORT sort_field @@ -903,6 +899,19 @@ public void testPartiallyMappedNonKeywordFieldsMarkedAsPotentiallyUnmapped() { } } + public void testWrapPartiallyUnmappedFieldWidensSmallNumerics() { + Set mappedIndices = Set.of("idx_mapped"); + for (DataType smallNumeric : List.of(DataType.SHORT, DataType.BYTE, DataType.FLOAT, DataType.HALF_FLOAT, DataType.SCALED_FLOAT)) { + EsField field = new EsField("f", smallNumeric, emptyMap(), true, EsField.TimeSeriesFieldType.NONE); + InvalidMappedField wrapped = (InvalidMappedField) IndexResolver.wrapPartiallyUnmappedField(field, "f", "f", mappedIndices); + assertThat( + "Partially-unmapped " + smallNumeric + " field should be stored under its widened type name", + wrapped.getTypesToIndices(), + equalTo(Map.of(smallNumeric.widenSmallNumeric().typeName(), mappedIndices)) + ); + } + } + public void testTbucketWithUnmappedTimestampWithLookupJoin() { var query = """ FROM test @@ -1146,13 +1155,16 @@ public void testDisallowLoadWithPartialNonKeywordAndTypeConflictInSameEval() { "conflicted", Map.of(DataType.LONG.typeName(), Set.of("idx_a"), DataType.DOUBLE.typeName(), Set.of("idx_b")) ); + var partialLong = InvalidMappedField.potentiallyUnmapped( + "partial_long", + Map.of(DataType.LONG.typeName(), Set.of("idx_a", "idx_b")) + ); var merged = new EsIndex( "idx*", - Map.of("partial_long", longField("partial_long"), "conflicted", conflicted), + Map.of("partial_long", partialLong, "conflicted", conflicted), Map.of("idx_a", IndexMode.STANDARD, "idx_b", IndexMode.STANDARD, "idx_unmapped", IndexMode.STANDARD), Map.of(), - Map.of(), - Map.of("partial_long", Set.of("idx_unmapped")) + Map.of() ); assertUnmappedLoadError( analyzer().addIndex("idx*", IndexResolution.valid(merged)), @@ -1187,13 +1199,13 @@ public void testAllowLoadCommaSeparatedIndicesWhenPartialNonKeywordUnused() { assumeTrue("Requires OPTIONAL_FIELDS_V5", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V5.isEnabled()); var pattern = "idx_a,idx_b"; + var partialLong = InvalidMappedField.potentiallyUnmapped("partial_long", Map.of(DataType.LONG.typeName(), Set.of("idx_a"))); var merged = new EsIndex( pattern, - Map.of("partial_long", longField("partial_long"), "common", keywordField("common")), + Map.of("partial_long", partialLong, "common", keywordField("common")), Map.of("idx_a", IndexMode.STANDARD, "idx_b", IndexMode.STANDARD), Map.of(), - Map.of(), - Map.of("partial_long", Set.of("idx_b")) + Map.of() ); var plan = analyzer().addIndex(pattern, IndexResolution.valid(merged)) .statement(setUnmappedLoad("FROM idx_a, idx_b | KEEP common")); @@ -1204,13 +1216,13 @@ public void testDisallowLoadCommaSeparatedIndicesWhenPartialNonKeywordUsed() { assumeTrue("Requires OPTIONAL_FIELDS_V5", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V5.isEnabled()); var pattern = "idx_a,idx_b"; + var partialLong = InvalidMappedField.potentiallyUnmapped("partial_long", Map.of(DataType.LONG.typeName(), Set.of("idx_a"))); var merged = new EsIndex( pattern, - Map.of("partial_long", longField("partial_long"), "common", keywordField("common")), + Map.of("partial_long", partialLong, "common", keywordField("common")), Map.of("idx_a", IndexMode.STANDARD, "idx_b", IndexMode.STANDARD), Map.of(), - Map.of(), - Map.of("partial_long", Set.of("idx_b")) + Map.of() ); assertUnmappedLoadError( analyzer().addIndex(pattern, IndexResolution.valid(merged)), @@ -1296,9 +1308,9 @@ public void testDisallowLoadWithPartiallyMappedNonKeywordInMvExpand() { public void testDisallowLoadWithPartiallyMappedNonKeywordDottedPath() { assumeTrue("Requires OPTIONAL_FIELDS_V5", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V5.isEnabled()); - var sub = longField("sub"); + var sub = InvalidMappedField.potentiallyUnmapped("sub", Map.of(DataType.LONG.typeName(), Set.of("idx_mapped"))); var obj = new EsField("obj", DataType.OBJECT, Map.of("sub", sub), true, EsField.TimeSeriesFieldType.NONE); - var esIndex = partialIndex(Map.of("obj", obj), Set.of("obj.sub")); + var esIndex = new EsIndex("idx*", Map.of("obj", obj), Map.of("idx_mapped", IndexMode.STANDARD), Map.of(), Map.of()); assertUnmappedLoadError(analyzer().addIndex(esIndex), "FROM idx* | SORT `obj.sub`", partiallyUnmappedNonKeywordError("obj.sub")); } @@ -1310,7 +1322,7 @@ public void testDisallowLoadWithPartialUnionTimestampInWhere() { assumeTrue("Requires OPTIONAL_FIELDS_V5", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V5.isEnabled()); var pattern = "sample_data,sample_data_ts_nanos,no_mapping_sample_data"; - var tsField = new InvalidMappedField( + var tsField = InvalidMappedField.potentiallyUnmapped( "@timestamp", Map.of(DataType.DATETIME.typeName(), Set.of("sample_data"), DataType.DATE_NANOS.typeName(), Set.of("sample_data_ts_nanos")) ); @@ -1326,8 +1338,7 @@ public void testDisallowLoadWithPartialUnionTimestampInWhere() { IndexMode.STANDARD ), Map.of(), - Map.of(), - Map.of("@timestamp", Set.of("no_mapping_sample_data")) + Map.of() ); assertUnmappedLoadError( analyzer().addIndex(pattern, IndexResolution.valid(merged)), @@ -1423,7 +1434,7 @@ private static TestAnalyzer test() { private static TestAnalyzer index1() { Map mapping = Map.of("field", new UnsupportedEsField("field", List.of("flattened"))); - return analyzer().addIndex(new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD), Map.of(), Map.of(), Map.of())); + return analyzer().addIndex(new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD), Map.of(), Map.of())); } private static void assertUnmappedLoadError(TestAnalyzer analyzer, String query, Matcher matcher) { @@ -1445,9 +1456,13 @@ private void typeConflictVerificationFailure(String statement, Map mapping, Set partialFieldNames) { - Map> fieldToUnmappedIndices = partialFieldNames.stream() - .collect(Collectors.toMap(f -> f, f -> Set.of("idx_unmapped"))); - return new EsIndex("idx*", mapping, Map.of("idx_mapped", IndexMode.STANDARD), Map.of(), Map.of(), fieldToUnmappedIndices); + Set mappedIndices = Set.of("idx_mapped"); + Map wrappedMapping = new HashMap<>(mapping); + for (String fieldName : partialFieldNames) { + EsField field = wrappedMapping.get(fieldName); + wrappedMapping.put(fieldName, IndexResolver.wrapPartiallyUnmappedField(field, fieldName, fieldName, mappedIndices)); + } + return new EsIndex("idx*", wrappedMapping, Map.of("idx_mapped", IndexMode.STANDARD), Map.of(), Map.of()); } private static EsField longField(String name) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexGenerator.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexGenerator.java index 2a686260e804a..50af3acdf0735 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexGenerator.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexGenerator.java @@ -25,19 +25,19 @@ public class EsIndexGenerator { public static EsIndex esIndex(String name) { - return new EsIndex(name, Map.of(), Map.of(), Map.of(), Map.of(), Map.of()); + return new EsIndex(name, Map.of(), Map.of(), Map.of(), Map.of()); } public static EsIndex esIndex(String name, Map mapping) { - return new EsIndex(name, mapping, Map.of(), Map.of(), Map.of(), Map.of()); + return new EsIndex(name, mapping, Map.of(), Map.of(), Map.of()); } public static EsIndex esIndex(String name, Map mapping, Map indexNameWithModes) { - return new EsIndex(name, mapping, indexNameWithModes, Map.of(), Map.of(), Map.of()); + return new EsIndex(name, mapping, indexNameWithModes, Map.of(), Map.of()); } public static EsIndex randomEsIndex() { - return new EsIndex(randomIdentifier(), randomMapping(), randomIndexNameWithModes(), Map.of(), Map.of(), Map.of()); + return new EsIndex(randomIdentifier(), randomMapping(), randomIndexNameWithModes(), Map.of(), Map.of()); } public static Map randomMapping() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/AbstractLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/AbstractLogicalPlanOptimizerTests.java index b8f243c6a3f37..91c634b3a6d2f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/AbstractLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/AbstractLogicalPlanOptimizerTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.session.IndexResolver; import org.junit.BeforeClass; import java.util.LinkedHashMap; @@ -95,17 +96,17 @@ protected static TestAnalyzer metricsAnalyzer() { protected static TestAnalyzer multiIndexAnalyzer() { var multiIndexMapping = loadMapping("mapping-basic.json"); + EsField partialTypeKeyword = new EsField("partial_type_keyword", KEYWORD, emptyMap(), true, EsField.TimeSeriesFieldType.NONE); multiIndexMapping.put( "partial_type_keyword", - new EsField("partial_type_keyword", KEYWORD, emptyMap(), true, EsField.TimeSeriesFieldType.NONE) + IndexResolver.wrapPartiallyUnmappedField(partialTypeKeyword, "partial_type_keyword", "partial_type_keyword", Set.of("test1")) ); var multiIndex = new EsIndex( "multi_index", multiIndexMapping, Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), Map.of(), - Map.of(), - Map.of("partial_type_keyword", Set.of("test2")) + Map.of() ); return analyzerWithEnrichPolicies().addIndex(multiIndex); } @@ -137,8 +138,7 @@ protected static TestAnalyzer unionIndexAnalyzer() { Map.of("languages", languages, "last_name", lastName, "salary_change", salaryChange, "first_name", firstName, "id", idField), Map.of("union_types_index", IndexMode.STANDARD, "union_types_index_incompatible", IndexMode.STANDARD), Map.of("", List.of("union_types_index*")), - Map.of("", List.of("union_types_index_incompatible", "union_types_index")), - Map.of() + Map.of("", List.of("union_types_index_incompatible", "union_types_index")) ); return analyzerWithEnrichPolicies().addAnalysisTestsInferenceResolution() .addIndex(unionIndex) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java index 070ec723eaa2b..dc194a6f9f467 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.esql.TestAnalyzer; import org.elasticsearch.xpack.esql.analysis.Analyzer; import org.elasticsearch.xpack.esql.analysis.PreAnalyzer; +import org.elasticsearch.xpack.esql.analysis.UnmappedResolution; import org.elasticsearch.xpack.esql.approximation.ApproximationPlan; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.FoldContext; @@ -44,6 +45,7 @@ import org.elasticsearch.xpack.esql.plan.EsqlStatement; import org.elasticsearch.xpack.esql.plan.IndexPattern; import org.elasticsearch.xpack.esql.plan.QueryPlan; +import org.elasticsearch.xpack.esql.plan.logical.Insist; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.physical.ExchangeExec; import org.elasticsearch.xpack.esql.plan.physical.ExchangeSinkExec; @@ -58,6 +60,7 @@ import org.elasticsearch.xpack.esql.plugin.QueryPragmas; import org.elasticsearch.xpack.esql.plugin.ReductionPlan; import org.elasticsearch.xpack.esql.session.Configuration; +import org.elasticsearch.xpack.esql.session.IndexResolver; import org.elasticsearch.xpack.esql.session.Versioned; import org.elasticsearch.xpack.esql.stats.SearchStats; import org.junit.internal.AssumptionViolatedException; @@ -85,7 +88,6 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -import static java.util.stream.Collectors.toSet; import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.CSV_DATASET; import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_PARSER; import static org.elasticsearch.xpack.esql.EsqlTestUtils.analyzer; @@ -254,13 +256,16 @@ private List> doTests() throws IOException { Path queryPath = PathUtils.get(basePath.toString(), queryPathParts); Files.createDirectories(queryPath.getParent()); Files.writeString(queryPath, esqlQuery); + UnmappedResolution unmappedResolution = statement.setting(UNMAPPED_FIELDS); TestAnalyzer testAnalyzer = analyzer().addLanguagesLookup() .addTestLookup() .addAnalysisTestsEnrichResolution() .addAnalysisTestsInferenceResolution() .minimumTransportVersion(transportVersion) - .unmappedResolution(statement.setting(UNMAPPED_FIELDS)); - loadIndexResolution(testDatasets(parsedPlan)).forEach( + .unmappedResolution(unmappedResolution); + boolean trackUnmappedFieldIndices = unmappedResolution == UnmappedResolution.LOAD + || parsedPlan.anyMatch(p -> p instanceof Insist); + loadIndexResolution(testDatasets(parsedPlan), trackUnmappedFieldIndices).forEach( (pattern, resolution) -> testAnalyzer.addIndex(pattern.indexPattern(), resolution) ); Analyzer analyzer = testAnalyzer.buildAnalyzer(); @@ -789,33 +794,38 @@ private static Map testD } public static Map loadIndexResolution( - Map datasets + Map datasets, + boolean trackUnmappedFieldIndices ) { Map indexResolutions = new HashMap<>(); for (var entry : datasets.entrySet()) { - indexResolutions.put(entry.getKey(), loadIndexResolution(entry.getValue())); + indexResolutions.put(entry.getKey(), loadIndexResolution(entry.getValue(), trackUnmappedFieldIndices)); } return indexResolutions; } + public static Map loadIndexResolution( + Map datasets + ) { + return loadIndexResolution(datasets, false); + } + public static IndexResolution loadIndexResolution(CsvTestsDataLoader.MultiIndexTestDataset datasets) { + return loadIndexResolution(datasets, false); + } + + public static IndexResolution loadIndexResolution( + CsvTestsDataLoader.MultiIndexTestDataset datasets, + boolean trackUnmappedFieldIndices + ) { var indexNames = datasets.datasets().stream().map(CsvTestsDataLoader.TestDataset::indexName); Map indexModes = indexNames.collect(Collectors.toMap(x -> x, x -> IndexMode.STANDARD)); List mappings = datasets.datasets() .stream() .map(ds -> new MappingPerIndex(ds.indexName(), createMappingForIndex(ds))) .toList(); - var mergedMappings = mergeMappings(mappings); - return IndexResolution.valid( - new EsIndex( - datasets.indexPattern(), - mergedMappings.mapping, - indexModes, - Map.of(), - Map.of(), - mergedMappings.fieldToUnmappedIndices - ) - ); + var mergedMappings = mergeMappings(mappings, trackUnmappedFieldIndices); + return IndexResolution.valid(new EsIndex(datasets.indexPattern(), mergedMappings.mapping, indexModes, Map.of(), Map.of())); } // TODO should de-duplicate, strong overlap with CsvTestsDataLoader#readMappingFile @@ -869,46 +879,39 @@ private static Map createMappingForIndex(CsvTestsDataLoader.Tes record MappingPerIndex(String index, Map mapping) {} - record MergedResult(Map mapping, Map> fieldToUnmappedIndices) {} + record MergedResult(Map mapping) {} - private static MergedResult mergeMappings(List mappingsPerIndex) { - int numberOfIndices = mappingsPerIndex.size(); - Set allIndices = mappingsPerIndex.stream().map(MappingPerIndex::index).collect(toSet()); - Map> columnNamesToFieldByIndices = new HashMap<>(); + private static MergedResult mergeMappings(List mappingsPerIndex, boolean trackUnmappedFieldIndices) { + Map> fieldNamesToFieldByIndices = new HashMap<>(); for (var mappingPerIndex : mappingsPerIndex) { for (var entry : mappingPerIndex.mapping().entrySet()) { - String columnName = entry.getKey(); - EsField field = entry.getValue(); - columnNamesToFieldByIndices.computeIfAbsent(columnName, k -> new HashMap<>()).put(mappingPerIndex.index(), field); - } - } - - Map> fieldToUnmappedIndices = new HashMap<>(); - for (var e : columnNamesToFieldByIndices.entrySet()) { - if (e.getValue().size() < numberOfIndices) { - Set unmappedIndices = allIndices.stream().filter(i -> e.getValue().containsKey(i) == false).collect(toSet()); - if (unmappedIndices.isEmpty() == false) { - fieldToUnmappedIndices.put(e.getKey(), unmappedIndices); - } + fieldNamesToFieldByIndices.computeIfAbsent(entry.getKey(), k -> new HashMap<>()) + .put(mappingPerIndex.index(), entry.getValue()); } } - var mappings = columnNamesToFieldByIndices.entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> mergeFields(e.getKey(), e.getValue()))); - return new MergedResult(mappings, fieldToUnmappedIndices); + int numberOfIndices = mappingsPerIndex.size(); + var mappings = fieldNamesToFieldByIndices.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> { + String fieldName = e.getKey(); + Map indexToFields = e.getValue(); + EsField field = mergeFields(fieldName, indexToFields); + return trackUnmappedFieldIndices + ? IndexResolver.wrapIfPartiallyUnmapped(field, fieldName, fieldName, indexToFields.keySet(), numberOfIndices) + : field; + })); + return new MergedResult(mappings); } - private static EsField mergeFields(String index, Map columnNameToField) { - var indexFields = columnNameToField.values(); - if (indexFields.stream().distinct().count() > 1) { + private static EsField mergeFields(String fieldName, Map indexToFields) { + var fields = indexToFields.values(); + if (fields.stream().distinct().count() > 1) { var typesToIndices = new HashMap>(); - for (var typeToIndex : columnNameToField.entrySet()) { - typesToIndices.computeIfAbsent(typeToIndex.getValue().getDataType().typeName(), k -> new HashSet<>()) - .add(typeToIndex.getKey()); + for (var indexToField : indexToFields.entrySet()) { + typesToIndices.computeIfAbsent(indexToField.getValue().getDataType().typeName(), k -> new HashSet<>()) + .add(indexToField.getKey()); } - return new InvalidMappedField(index, typesToIndices); + return new InvalidMappedField(fieldName, typesToIndices); } else { - return indexFields.iterator().next(); + return fields.iterator().next(); } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index f5c82f900cbb3..e91272b2a7ac8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -42,7 +42,7 @@ import org.elasticsearch.xpack.esql.core.expression.UnsupportedAttribute; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; +import org.elasticsearch.xpack.esql.core.type.UnionTypeEsField; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.expression.Order; import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; @@ -2615,7 +2615,7 @@ public void testSortWithLimitBy() { } private boolean isMultiTypeEsField(Expression e) { - return e instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField; + return e instanceof FieldAttribute fa && fa.field() instanceof UnionTypeEsField; } private Stat queryStatsFor(PhysicalPlan plan) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 0bbb9b23b8120..b0ef9c48615cd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -11203,7 +11203,6 @@ public void testTsWildcardStatsWithMixedIndexModes() { mapping, Map.of("ts_index", IndexMode.TIME_SERIES, "standard_index", IndexMode.STANDARD), Map.of(), - Map.of(), Map.of() ); var testAnalyzer = EsqlTestUtils.analyzer().addIndex(IndexResolution.valid(mixedIndex)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java index 28269a5d87f2d..6907856ad932e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java @@ -60,7 +60,7 @@ public void testRefineConcreteTimeSeriesResolutionReturnsHelpfulError() { public void testRefineConcreteTimeSeriesResolutionKeepsOriginalFailures() { FieldCapabilitiesFailure failure = new FieldCapabilitiesFailure(new String[] { "logs" }, new ElasticsearchException("boom")); IndexResolution originalResolution = IndexResolution.valid( - new EsIndex("logs", Map.of(), Map.of(), Map.of(), Map.of(), Map.of()), + new EsIndex("logs", Map.of(), Map.of(), Map.of(), Map.of()), Set.of(), Map.of("remote", List.of(failure)) ); @@ -76,7 +76,7 @@ public void testRefineConcreteTimeSeriesResolutionKeepsOriginalFailures() { private static IndexResolution resolvedIndex(String indexName) { return IndexResolution.valid( - new EsIndex(indexName, Map.of(), Map.of(indexName, IndexMode.STANDARD), Map.of(), Map.of(), Map.of()), + new EsIndex(indexName, Map.of(), Map.of(indexName, IndexMode.STANDARD), Map.of(), Map.of()), Set.of(indexName), Map.of() ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/CompactInvalidMappedFieldTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/CompactInvalidMappedFieldTests.java new file mode 100644 index 0000000000000..086f4c349ef97 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/CompactInvalidMappedFieldTests.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.type; + +import org.elasticsearch.core.Strings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.core.type.CompactInvalidMappedField; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; + +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.elasticsearch.test.MapMatcher.assertMap; +import static org.elasticsearch.test.MapMatcher.matchesMap; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class CompactInvalidMappedFieldTests extends ESTestCase { + public void testKeepsAllIndicesWhenAtOrBelowLimit() { + Map> input = Map.of( + DataType.KEYWORD, + new LinkedHashSet<>(Set.of("idx_a", "idx_b")), + DataType.LONG, + new LinkedHashSet<>(Set.of("idx_c", "idx_d", "idx_e")) + ); + + assertMap( + new CompactInvalidMappedField("f", input).getTypesToIndices(), + matchesMap().entry(DataType.KEYWORD.typeName(), Set.of("idx_a", "idx_b")) + .entry(DataType.LONG.typeName(), Set.of("idx_c", "idx_d", "idx_e")) + ); + } + + public void testTruncatesAboveLimitAndAddsEllipsisSentinel() { + Set manyIndices = IntStream.range(0, 5_000) + .mapToObj(i -> Strings.format("idx_%05d", i)) + .collect(Collectors.toCollection(LinkedHashSet::new)); + + CompactInvalidMappedField field = new CompactInvalidMappedField("f", Map.of(DataType.KEYWORD, manyIndices)); + + assertMap( + field.getTypesToIndices(), + matchesMap().entry(DataType.KEYWORD.typeName(), Set.of("idx_00000", "idx_00001", "idx_00002", "...")) + ); + } + + public void testErrorMessageReflectsFullInputCountEvenAfterTruncation() { + Set manyIndices = IntStream.range(0, 5_000) + .mapToObj(i -> Strings.format("idx_%05d", i)) + .collect(Collectors.toCollection(LinkedHashSet::new)); + Map> input = new TreeMap<>(Map.of(DataType.KEYWORD, manyIndices)); + + String message = new CompactInvalidMappedField("f", input).errorMessage(); + + assertThat(message, containsString("[1] incompatible types")); + assertThat(message, containsString("[idx_00000, idx_00001, idx_00002]")); + assertThat(message, containsString("[" + (5_000 - 3) + "] other indices")); + } + + public void testErrorMessageMatchesInvalidMappedFieldForSmallInputs() { + Set kwIndices = new LinkedHashSet<>(Set.of("idx_a", "idx_b")); + Set longIndices = new LinkedHashSet<>(Set.of("idx_c")); + Map> compactInput = new TreeMap<>(Map.of(DataType.KEYWORD, kwIndices, DataType.LONG, longIndices)); + Map> legacyInput = new TreeMap<>( + Map.of(DataType.KEYWORD.typeName(), kwIndices, DataType.LONG.typeName(), longIndices) + ); + + assertThat( + new CompactInvalidMappedField("f", compactInput).errorMessage(), + equalTo(new InvalidMappedField("f", legacyInput).errorMessage()) + ); + } + + public void testPotentiallyUnmappedFlagAndMessageInsistOnKeyword() { + Map> input = new TreeMap<>(Map.of(DataType.LONG, new LinkedHashSet<>(Set.of("idx_a")))); + + CompactInvalidMappedField field = CompactInvalidMappedField.potentiallyUnmapped("f", input); + + assertThat(field.isPotentiallyUnmapped(), equalTo(true)); + assertThat(field.errorMessage(), containsString("[keyword] due to loading from _source")); + assertMap(field.getTypesToIndices(), matchesMap().entry(DataType.LONG.typeName(), Set.of("idx_a"))); + } + + public void testTypesReflectsKeysOfTruncatedMap() { + Map> input = new TreeMap<>( + Map.of( + DataType.KEYWORD, + IntStream.range(0, 100).mapToObj(i -> "k" + i).collect(Collectors.toSet()), + DataType.LONG, + Set.of("only") + ) + ); + + assertThat(new CompactInvalidMappedField("f", input).types(), containsInAnyOrder(DataType.KEYWORD, DataType.LONG)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/CompactMultiTypeEsFieldTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/CompactMultiTypeEsFieldTests.java new file mode 100644 index 0000000000000..317718b01a3f4 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/CompactMultiTypeEsFieldTests.java @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.type; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.CompactMultiTypeEsField; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.expression.ExpressionWritables; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToBoolean; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianPoint; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianShape; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatetime; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToGeoPoint; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToGeoShape; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToIpLeadingZerosRejected; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToLong; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToString; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToVersion; +import org.elasticsearch.xpack.esql.session.Configuration; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration; + +/** + * Mirror of {@link MultiTypeEsFieldTests} for the type-keyed {@link CompactMultiTypeEsField}. + */ +public class CompactMultiTypeEsFieldTests extends AbstractEsFieldTypeTests { + private Configuration config; + + @Before + public void initConfig() { + config = randomConfiguration(); + } + + @Override + protected Configuration config() { + return config; + } + + @Override + protected CompactMultiTypeEsField createTestInstance() { + String name = randomAlphaOfLength(4); + boolean toString = randomBoolean(); + DataType dataType = randomFrom(types()); + DataType toType = toString ? DataType.KEYWORD : dataType; + Map typeToConvertExpressions = randomConvertExpressions(name, toString, dataType); + Expression unmappedConversionExpression = randomBoolean() ? null : createToString(name, dataType); + + EsField.TimeSeriesFieldType tsType = randomFrom(EsField.TimeSeriesFieldType.values()); + + return new CompactMultiTypeEsField(name, toType, false, typeToConvertExpressions, tsType, unmappedConversionExpression); + } + + @Override + protected CompactMultiTypeEsField mutateInstance(CompactMultiTypeEsField instance) throws IOException { + String name = instance.getName(); + DataType dataType = instance.getDataType(); + Map typeToConvertExpressions = instance.getTypeToConversionExpressions(); + EsField.TimeSeriesFieldType tsType = instance.getTimeSeriesFieldType(); + Expression unmappedConversionExpression = instance.getUnmappedConversionExpression(); + switch (between(0, 4)) { + case 0 -> name = randomAlphaOfLength(name.length() + 1); + case 1 -> dataType = randomValueOtherThan(dataType, () -> randomFrom(DataType.types())); + case 2 -> typeToConvertExpressions = mutateConvertExpressions(name, dataType, typeToConvertExpressions); + case 3 -> tsType = randomValueOtherThan(tsType, () -> randomFrom(EsField.TimeSeriesFieldType.values())); + case 4 -> unmappedConversionExpression = unmappedConversionExpression != null ? null : createToString(name, dataType); + default -> throw new IllegalArgumentException(); + } + return new CompactMultiTypeEsField(name, dataType, false, typeToConvertExpressions, tsType, unmappedConversionExpression); + } + + @Override + protected final NamedWriteableRegistry getNamedWriteableRegistry() { + List entries = new ArrayList<>(ExpressionWritables.allExpressions()); + entries.addAll(ExpressionWritables.unaryScalars()); + return new NamedWriteableRegistry(entries); + } + + /** + * Random map keyed by source {@link DataType}, so it can be used as the source-type map of + * {@link CompactMultiTypeEsField}. + */ + private Map randomConvertExpressions(String name, boolean toString, DataType dataType) { + Map typeToConvertExpressions = new HashMap<>(); + if (toString) { + typeToConvertExpressions.put(DataType.KEYWORD, createToString(name, DataType.KEYWORD)); + typeToConvertExpressions.put(dataType, createToString(name, dataType)); + } else { + typeToConvertExpressions.put(DataType.KEYWORD, testConvertExpression(name, DataType.KEYWORD, dataType)); + typeToConvertExpressions.put(dataType, testConvertExpression(name, dataType, dataType)); + } + return typeToConvertExpressions; + } + + private Map mutateConvertExpressions( + String name, + DataType toType, + Map typeToConvertExpressions + ) { + return randomValueOtherThan( + typeToConvertExpressions, + () -> randomConvertExpressions(name, toType == DataType.KEYWORD, randomFrom(types())) + ); + } + + private static List types() { + return List.of( + DataType.BOOLEAN, + DataType.DATETIME, + DataType.DOUBLE, + DataType.FLOAT, + DataType.INTEGER, + DataType.IP, + DataType.KEYWORD, + DataType.LONG, + DataType.GEO_POINT, + DataType.GEO_SHAPE, + DataType.CARTESIAN_POINT, + DataType.CARTESIAN_SHAPE, + DataType.VERSION + ); + } + + private Expression testConvertExpression(String name, DataType fromType, DataType toType) { + FieldAttribute fromField = fieldAttribute(name, fromType); + return switch (toType) { + case BOOLEAN -> new ToBoolean(Source.EMPTY, fromField); + case DATETIME -> new ToDatetime(Source.EMPTY, fromField, config()); + case DOUBLE, FLOAT -> new ToDouble(Source.EMPTY, fromField); + case INTEGER -> new ToInteger(Source.EMPTY, fromField); + case LONG -> new ToLong(Source.EMPTY, fromField); + case IP -> new ToIpLeadingZerosRejected(Source.EMPTY, fromField); + case KEYWORD, TEXT -> new ToString(Source.EMPTY, fromField, config()); + case GEO_POINT -> new ToGeoPoint(Source.EMPTY, fromField); + case GEO_SHAPE -> new ToGeoShape(Source.EMPTY, fromField); + case CARTESIAN_POINT -> new ToCartesianPoint(Source.EMPTY, fromField); + case CARTESIAN_SHAPE -> new ToCartesianShape(Source.EMPTY, fromField); + case VERSION -> new ToVersion(Source.EMPTY, fromField); + default -> throw new UnsupportedOperationException("Conversion from " + fromType + " to " + toType + " is not supported"); + }; + } + + private static FieldAttribute fieldAttribute(String name, DataType dataType) { + return new FieldAttribute(Source.EMPTY, name, new EsField(name, dataType, Map.of(), true, EsField.TimeSeriesFieldType.NONE)); + } + + private ToString createToString(String name, DataType dataType) { + return new ToString(Source.EMPTY, fieldAttribute(name, dataType), config()); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/MultiTypeEsFieldMemoryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/MultiTypeEsFieldMemoryTests.java new file mode 100644 index 0000000000000..9b66f779e765f --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/MultiTypeEsFieldMemoryTests.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.type; + +import org.apache.lucene.tests.util.RamUsageTester; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.TransportVersionUtils; +import org.elasticsearch.xpack.esql.core.type.CompactInvalidMappedField; +import org.elasticsearch.xpack.esql.core.type.CompactMultiTypeEsField; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; +import org.elasticsearch.xpack.esql.index.EsIndex; +import org.elasticsearch.xpack.esql.index.IndexResolution; +import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; + +import java.lang.reflect.Field; +import java.time.ZoneId; +import java.util.Collection; +import java.util.HashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.elasticsearch.xpack.esql.EsqlTestUtils.analyzer; +import static org.elasticsearch.xpack.esql.core.type.CompactMultiTypeEsField.CompactMultiTypeEsField; +import static org.hamcrest.Matchers.lessThan; + +/** + * End-to-end check that an analyzed plan over many union-typed fields, each conflicting across thousands of indices, retains substantially + * less memory under {@link CompactMultiTypeEsField} (paired with {@link CompactInvalidMappedField}'s truncated index lists) than under + * the legacy {@link InvalidMappedField} (keyed per-index). + */ +public class MultiTypeEsFieldMemoryTests extends ESTestCase { + private static final int NUM_INDICES = 5_000; + private static final int NUM_CONFLICTING_FIELDS = 50; + + /** + * {@link RamUsageTester} walks reflectively, which fails on JDK-internal classes (e.g. {@code sun.util.locale.BaseLocale}) that + * aren't opened to unnamed modules. The plan transitively references a {@link Locale} and a {@link ZoneId} via the analyzer's + * {@code Configuration}, so we treat those as opaque as they're irrelevant to the union-type memory we care about here. + */ + private static final RamUsageTester.Accumulator ACCUMULATOR = new RamUsageTester.Accumulator() { + @Override + public long accumulateObject(Object o, long shallowSize, Map fieldValues, Collection queue) { + return o instanceof Locale || o instanceof ZoneId ? shallowSize : super.accumulateObject(o, shallowSize, fieldValues, queue); + } + }; + + public void testV2AnalyzedPlanIsAtLeastTenTimesSmallerThanLegacy() { + String evalAssignments = IntStream.range(0, NUM_CONFLICTING_FIELDS) + .mapToObj(i -> "id_" + i + "_kw = id_" + i + "::keyword") + .collect(Collectors.joining(", ")); + String keepFields = IntStream.range(0, NUM_CONFLICTING_FIELDS).mapToObj(i -> "id_" + i + "_kw").collect(Collectors.joining(", ")); + String query = "FROM idx* | EVAL " + evalAssignments + " | KEEP " + keepFields + " | LIMIT 1"; + + assertThat(getBytesUsed(true, query) * 10L, lessThan(getBytesUsed(false, query))); + } + + private static long getBytesUsed(boolean compact, String query) { + TransportVersion transportVersion = compact + ? TransportVersionUtils.randomVersionSupporting(CompactMultiTypeEsField) + : TransportVersionUtils.randomVersionNotSupporting(CompactMultiTypeEsField); + LogicalPlan plan = analyzer().addIndex(unionTypedIndex(compact)).minimumTransportVersion(transportVersion).query(query); + return RamUsageTester.ramUsed(plan, ACCUMULATOR); + } + + private static IndexResolution unionTypedIndex(boolean compact) { + Map indexNamesWithModes = IntStream.range(0, NUM_INDICES) + .boxed() + .collect(Collectors.toMap(i -> "idx_" + i, i -> IndexMode.STANDARD)); + Map mapping = IntStream.range(0, NUM_CONFLICTING_FIELDS) + .boxed() + .collect(Collectors.toMap(i -> "id_" + i, i -> getEsField(compact, "id_" + i))); + return IndexResolution.valid(new EsIndex("idx*", mapping, indexNamesWithModes, Map.of(), Map.of())); + } + + private static EsField getEsField(boolean compact, String fieldName) { + Set keywordIndices = new HashSet<>(); + Set intIndices = new HashSet<>(); + for (int j = 0; j < NUM_INDICES; j++) { + (j % 2 == 0 ? keywordIndices : intIndices).add("idx_" + j); + } + return compact + ? new CompactInvalidMappedField(fieldName, Map.of(DataType.KEYWORD, keywordIndices, DataType.INTEGER, intIndices)) + : new InvalidMappedField( + fieldName, + Map.of(DataType.KEYWORD.typeName(), keywordIndices, DataType.INTEGER.typeName(), intIndices) + ); + } +}