From fb7ca87f8e0a21ad660c65783cd28821b854e744 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Tue, 13 Oct 2026 01:37:25 +0100 Subject: [PATCH 1/3] SOLR-9957: /export supports document transformers in fl * Parse fl with SolrReturnFields instead of splitting on commas, so transformers, their local params, and field aliases work * When fl has a transformer, collect each doc's fields into a SolrDocument, apply the transformer, and write the wanted fields; the plain path is unchanged * Transformer output is visible to the local streaming expression (expr) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../SOLR-9957-exportTransformers.yml | 10 ++ .../solr/handler/export/ExportWriter.java | 146 ++++++++++++++---- .../handler/export/ExportWriterStream.java | 13 +- .../solr/handler/export/TestExportWriter.java | 69 +++++++++ .../pages/exporting-result-sets.adoc | 3 + 5 files changed, 202 insertions(+), 39 deletions(-) create mode 100644 changelog/unreleased/SOLR-9957-exportTransformers.yml diff --git a/changelog/unreleased/SOLR-9957-exportTransformers.yml b/changelog/unreleased/SOLR-9957-exportTransformers.yml new file mode 100644 index 000000000000..aa37e8f4183b --- /dev/null +++ b/changelog/unreleased/SOLR-9957-exportTransformers.yml @@ -0,0 +1,10 @@ +title: > + The /export handler now supports document transformers (e.g. `[docid]`) and field aliases in `fl`, + and their output is visible to its local streaming expression (`expr`). An aliased field such as + `fl=x:myfield` is now returned as `x`; previously it was silently returned as `myfield`. +type: added +authors: + - name: David Smiley +links: + - name: SOLR-9957 + url: https://issues.apache.org/jira/browse/SOLR-9957 diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java index 7b82ad28b14c..866e497ffce6 100644 --- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java +++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java @@ -26,6 +26,7 @@ import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -36,6 +37,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Query; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.util.BitSetIterator; @@ -52,10 +54,12 @@ import org.apache.solr.common.MapWriter; import org.apache.solr.common.MapWriter.EntryWriter; import org.apache.solr.common.PushWriter; +import org.apache.solr.common.SolrDocument; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.params.StreamParams; import org.apache.solr.common.util.JavaBinCodec; +import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.SolrCore; import org.apache.solr.metrics.SolrMetricsContext; import org.apache.solr.request.SolrQueryRequest; @@ -64,7 +68,9 @@ import org.apache.solr.response.JacksonJsonWriter; import org.apache.solr.response.JavaBinResponseWriter; import org.apache.solr.response.QueryResponseWriter; +import org.apache.solr.response.ResultContext; import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.response.transform.DocTransformer; import org.apache.solr.schema.BoolField; import org.apache.solr.schema.DateValueFieldType; import org.apache.solr.schema.DoubleValueFieldType; @@ -76,7 +82,10 @@ import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.SortableTextField; import org.apache.solr.schema.StrField; +import org.apache.solr.search.DocIterationInfo; +import org.apache.solr.search.DocList; import org.apache.solr.search.DocValuesIteratorCache; +import org.apache.solr.search.ReturnFields; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrReturnFields; import org.apache.solr.search.SortSpec; @@ -124,7 +133,9 @@ public void write(SortDoc sortDoc, LeafReaderContext readerContext, EntryWriter private int priorityQueueSize; StreamExpression streamExpression; StreamContext streamContext; + SolrReturnFields returnFields; List fieldWriters; + DocTransformer transformer; int totalHits = 0; FixedBitSet[] sets = null; PushWriter writer; @@ -268,35 +279,38 @@ private void _write(OutputStream os) throws IOException { } } SolrParams params = req.getParams(); - String fl = params.get("fl"); - String[] fields = null; - - if (fl == null) { + if (params.get(CommonParams.FL) == null) { writeException( (new IOException(new SyntaxError("export field list (fl) must be specified."))), writer, true); return; - } else { - fields = fl.split(","); - - for (int i = 0; i < fields.length; i++) { + } - fields[i] = fields[i].trim(); + try { + returnFields = new SolrReturnFields(req); + } catch (Exception e) { + writeException(e, writer, true); + return; + } - if (fields[i].equals("score")) { - writeException( - (new IOException(new SyntaxError("Scoring is not currently supported with xsort."))), - writer, - true); - return; - } - } + if (returnFields.wantsScore()) { + writeException( + (new IOException(new SyntaxError("Scoring is not currently supported with xsort."))), + writer, + true); + return; } try { - fieldWriters = getFieldWriters(fields, req); + fieldWriters = getFieldWriters(returnFields, req); + transformer = returnFields.getTransformer(); + if (transformer != null) { + transformer.setContext( + new ExportResultContext( + returnFields, searcher, info.getResponseBuilder().getQuery(), req)); + } } catch (Exception e) { writeException(e, writer, true); return; @@ -483,33 +497,70 @@ void writeDoc( throws IOException { int ord = sortDoc.ord; LeafReaderContext context = leaves.get(ord); + if (transformer == null) { + for (FieldWriter fieldWriter : writers) { + fieldWriter.write(sortDoc, context, ew); + } + return; + } + + SolrDocument doc = new SolrDocument(); + EntryWriter docWriter = + new EntryWriter() { + @Override + public EntryWriter put(CharSequence k, Object v) { + doc.setField(k.toString(), materialize(v)); + return this; + } + }; for (FieldWriter fieldWriter : writers) { - fieldWriter.write(sortDoc, context, ew); + fieldWriter.write(sortDoc, context, docWriter); + } + transformer.transform(doc, sortDoc.docBase + sortDoc.docId, DocIterationInfo.NONE); + for (Map.Entry entry : doc) { + if (returnFields.wantsField(entry.getKey())) { + ew.put(entry.getKey(), entry.getValue()); + } } } - public List getFieldWriters(String[] fields, SolrQueryRequest req) + /** Converts lazily written values to plain Java objects that can outlive the write call. */ + static Object materialize(Object v) { + if (v instanceof IteratorWriter iteratorWriter) { + List lst = new ArrayList<>(); + iteratorWriter.toList(lst); + return lst; + } else if (v instanceof MapWriter mapWriter) { + return new SimpleOrderedMap<>(mapWriter); + } + return v; + } + + List getFieldWriters(SolrReturnFields solrReturnFields, SolrQueryRequest req) throws IOException { DocValuesIteratorCache dvIterCache = new DocValuesIteratorCache(req.getSearcher(), false); - SolrReturnFields solrReturnFields = new SolrReturnFields(fields, req); boolean includeStoredFields = req.getParams().getBool(INCLUDE_STORED_FIELDS_PARAM, false); + // Explicitly named fields, including any a transformer needs (e.g. the source of an alias) + Set requestFieldNames = new HashSet<>(); + if (solrReturnFields.getRequestedFieldNames() != null) { + requestFieldNames.addAll(solrReturnFields.getRequestedFieldNames()); + } + if (solrReturnFields.getLuceneFieldNames() != null) { + requestFieldNames.addAll(solrReturnFields.getLuceneFieldNames()); + } + List writers = new ArrayList<>(); Set docValueFields = new LinkedHashSet<>(); Map storedFields = new LinkedHashMap<>(); for (String field : req.getSearcher().getFieldNames()) { - if (!solrReturnFields.wantsField(field)) { + if (!solrReturnFields.wantsField(field) && !requestFieldNames.contains(field)) { continue; } SchemaField schemaField = req.getSchema().getField(field); FieldType fieldType = schemaField.getType(); - Set requestFieldNames = - solrReturnFields.getRequestedFieldNames() == null - ? Set.of() - : solrReturnFields.getRequestedFieldNames(); - if (canUseDocValues(schemaField, fieldType)) { // Prefer DocValues when available docValueFields.add(schemaField); @@ -610,6 +661,47 @@ public List getFieldWriters(String[] fields, SolrQueryRequest req) return writers; } + /** Context for transformers in {@code fl}; there is no {@link DocList} when exporting. */ + private static final class ExportResultContext extends ResultContext { + private final ReturnFields returnFields; + private final SolrIndexSearcher searcher; + private final Query query; + private final SolrQueryRequest req; + + ExportResultContext( + ReturnFields returnFields, SolrIndexSearcher searcher, Query query, SolrQueryRequest req) { + this.returnFields = returnFields; + this.searcher = searcher; + this.query = query; + this.req = req; + } + + @Override + public DocList getDocList() { + return null; + } + + @Override + public ReturnFields getReturnFields() { + return returnFields; + } + + @Override + public SolrIndexSearcher getSearcher() { + return searcher; + } + + @Override + public Query getQuery() { + return query; + } + + @Override + public SolrQueryRequest getRequest() { + return req; + } + } + private static boolean canUseDocValues(SchemaField schemaField, FieldType fieldType) { return schemaField.hasDocValues() // Special handling for SortableTextField: unlike other field types, it requires diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.java index 86bbefb8e37d..3c21cb8202cf 100644 --- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.java +++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.java @@ -18,7 +18,6 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; -import java.util.ArrayList; import java.util.List; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.TimeoutException; @@ -35,9 +34,6 @@ import org.apache.solr.client.solrj.io.stream.expr.StreamExpression; import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter; import org.apache.solr.client.solrj.io.stream.expr.StreamFactory; -import org.apache.solr.common.IteratorWriter; -import org.apache.solr.common.MapWriter; -import org.apache.solr.common.util.SimpleOrderedMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,14 +61,7 @@ private static final class TupleEntryWriter implements EntryWriter { @Override public EntryWriter put(CharSequence k, Object v) throws IOException { - if (v instanceof IteratorWriter) { - List lst = new ArrayList<>(); - ((IteratorWriter) v).toList(lst); - v = lst; - } else if (v instanceof MapWriter) { - v = new SimpleOrderedMap<>((MapWriter) v); - } - tuple.put(k.toString(), v); + tuple.put(k.toString(), ExportWriter.materialize(v)); return this; } } diff --git a/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java b/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java index 61384d505538..5be050c80632 100644 --- a/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java +++ b/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java @@ -1308,6 +1308,75 @@ public void testGlobFields() throws Exception { jq.contains("\"sortabledv\"")); } + @Test + @SuppressWarnings("unchecked") + public void testDocTransformers() throws Exception { + assertU(delQ("*:*")); + assertU(adoc("id", "1", "intdv", "10", "intdv_m", "1", "intdv_m", "2")); + assertU(adoc("id", "2", "intdv", "20")); + assertU(commit()); + assertU(adoc("id", "3", "intdv", "30")); + assertU(commit()); + + Map selectDocIds = new HashMap<>(); + String selectRsp = h.query(req("q", "*:*", "fl", "id,[docid]", "wt", "json")); + for (Map doc : + (List>) + Utils.getObjectByPath( + mapper.readValue(selectRsp, Map.class), false, "/response/docs")) { + selectDocIds.put((String) doc.get("id"), doc.get("[docid]")); + } + assertEquals(3, selectDocIds.size()); + + // [docid] matches /select's, and fields are materialized (incl. multiValued) + List> docs = + (List>) + queryJsonReturnDocs( + reqWithPath("/export", "q", "*:*", "fl", "id,intdv_m,[docid]", "sort", "id asc")); + assertEquals(3, docs.size()); + for (Map doc : docs) { + assertEquals(doc.toString(), selectDocIds.get(doc.get("id")), doc.get("[docid]")); + } + assertEquals(List.of(1, 2), docs.get(0).get("intdv_m")); + + // alias renames; transformer local params may contain commas + docs = + (List>) + queryJsonReturnDocs( + reqWithPath( + "/export", "q", "*:*", "fl", "id,x:intdv,v:[value v='a,b']", "sort", "id asc")); + assertEquals(3, docs.size()); + assertEquals(Map.of("id", "1", "x", 10, "v", "a,b"), docs.get(0)); + + // transformed fields are visible to streaming expressions + docs = + (List>) + queryJsonReturnDocs( + reqWithPath( + "/export", + "q", + "*:*", + "fl", + "id,[docid]", + "sort", + "id asc", + "expr", + "top(n=2,input(),sort=\"id desc\")")); + assertEquals(2, docs.size()); + assertEquals("3", docs.get(0).get("id")); + assertEquals(selectDocIds.get("3"), docs.get(0).get("[docid]")); + + // score is still rejected + docs = + (List>) + queryJsonReturnDocs( + reqWithPath("/export", "q", "*:*", "fl", "id,score", "sort", "id asc")); + assertEquals(1, docs.size()); + assertTrue( + docs.get(0).toString(), + docs.get(0).get(StreamParams.EXCEPTION).toString().contains("Scoring")); + } + @SuppressWarnings("rawtypes") private void validateSort(int numDocs) throws Exception { // 10 fields diff --git a/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc b/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc index 503d12eb2727..ab540eeefbed 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc @@ -101,6 +101,9 @@ Note that sort fields still require docValues, regardless of this setting. Wildcard patterns can be used for the field list (e.g. `fl=*_i`) and will be expanded to the list of fields that match the pattern and are able to be exported, see <>. +Field aliases (e.g. `fl=id,price:price_d`) and xref:document-transformers.adoc[document transformers] (e.g. `fl=id,[docid]`) are supported, and their output is also visible to a local streaming expression. +Transformers are applied one document at a time, and those that depend on scores or on a list of matching documents won't work. + Returning scores is not supported at this time. === Specifying the Local Streaming Expression From a7ee6ac595ebebdfe0210ded8e5b24d8bf31d473 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Tue, 13 Oct 2026 02:00:56 +0100 Subject: [PATCH 2/3] SOLR-9957: clarify which fields getFieldWriters fetches vs errors on Split the combined field-name set into two named sets, one for fetching transformer-read fields and one for deciding whether a non-exportable field errors or is skipped. No behavior change. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../solr/handler/export/ExportWriter.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java index 866e497ffce6..7872a61be1f5 100644 --- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java +++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java @@ -541,13 +541,16 @@ List getFieldWriters(SolrReturnFields solrReturnFields, SolrQueryRe DocValuesIteratorCache dvIterCache = new DocValuesIteratorCache(req.getSearcher(), false); boolean includeStoredFields = req.getParams().getBool(INCLUDE_STORED_FIELDS_PARAM, false); - // Explicitly named fields, including any a transformer needs (e.g. the source of an alias) - Set requestFieldNames = new HashSet<>(); + // Also fetch fields only a transformer reads, e.g. the f= of [geo]; wantsField excludes them + Set fieldsToFetch = + solrReturnFields.getLuceneFieldNames() != null + ? solrReturnFields.getLuceneFieldNames() + : Set.of(); + // Non-exportable fields named here error; ones matched only by a glob are skipped. + // Includes fieldsToFetch, since requested names hold an alias, not its source field. + Set explicitlyNamedFields = new HashSet<>(fieldsToFetch); if (solrReturnFields.getRequestedFieldNames() != null) { - requestFieldNames.addAll(solrReturnFields.getRequestedFieldNames()); - } - if (solrReturnFields.getLuceneFieldNames() != null) { - requestFieldNames.addAll(solrReturnFields.getLuceneFieldNames()); + explicitlyNamedFields.addAll(solrReturnFields.getRequestedFieldNames()); } List writers = new ArrayList<>(); @@ -555,7 +558,7 @@ List getFieldWriters(SolrReturnFields solrReturnFields, SolrQueryRe Map storedFields = new LinkedHashMap<>(); for (String field : req.getSearcher().getFieldNames()) { - if (!solrReturnFields.wantsField(field) && !requestFieldNames.contains(field)) { + if (!solrReturnFields.wantsField(field) && !fieldsToFetch.contains(field)) { continue; } SchemaField schemaField = req.getSchema().getField(field); @@ -568,7 +571,7 @@ List getFieldWriters(SolrReturnFields solrReturnFields, SolrQueryRe // Field is stored-only (no usable DocValues) if (includeStoredFields) { storedFields.put(field, schemaField); - } else if (requestFieldNames.contains(field)) { + } else if (explicitlyNamedFields.contains(field)) { // Explicitly requested field without DocValues and includeStoredFields=false throw new IOException( schemaField @@ -576,7 +579,7 @@ List getFieldWriters(SolrReturnFields solrReturnFields, SolrQueryRe + "Try setting includeStoredFields=true to retrieve this field from stored values."); } // Else: glob matched stored-only field without includeStoredFields - silently skip - } else if (requestFieldNames.contains(field)) { + } else if (explicitlyNamedFields.contains(field)) { // Explicitly requested field that has neither DocValues nor stored if (fieldType instanceof SortableTextField && !schemaField.useDocValuesAsStored()) { throw new IOException( From fbb2e7346302a945663e3b91b6d331e12d0bf576 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Wed, 14 Oct 2026 00:02:10 +0100 Subject: [PATCH 3/3] ref guide tweak --- .../modules/query-guide/pages/exporting-result-sets.adoc | 1 + 1 file changed, 1 insertion(+) diff --git a/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc b/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc index ab540eeefbed..823bc51e1612 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc @@ -103,6 +103,7 @@ Wildcard patterns can be used for the field list (e.g. `fl=*_i`) and will be exp Field aliases (e.g. `fl=id,price:price_d`) and xref:document-transformers.adoc[document transformers] (e.g. `fl=id,[docid]`) are supported, and their output is also visible to a local streaming expression. Transformers are applied one document at a time, and those that depend on scores or on a list of matching documents won't work. +Some transformers, like `[subquery]` and `[child]`, produce output that might not work well, or at all, with `/export` and streaming expressions. Returning scores is not supported at this time.