SOLR-9957: /export supports document transformers in fl

changelog/unreleased/SOLR-9957-exportTransformers.yml

@@ -0,0 +1,10 @@
+title: >
+  The /export handler now supports document transformers (e.g. `[docid]`) and field aliases in `fl`,
+  and their output is visible to its local streaming expression (`expr`). An aliased field such as
+  `fl=x:myfield` is now returned as `x`; previously it was silently returned as `myfield`.
+type: added
+authors:
+  - name: David Smiley
+links:
+  - name: SOLR-9957
+    url: https://issues.apache.org/jira/browse/SOLR-9957

solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java

@@ -26,6 +26,7 @@
 import java.lang.invoke.MethodHandles;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -36,6 +37,7 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.BitSetIterator;
@@ -52,10 +54,12 @@
 import org.apache.solr.common.MapWriter;
 import org.apache.solr.common.MapWriter.EntryWriter;
 import org.apache.solr.common.PushWriter;
+import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.params.StreamParams;
 import org.apache.solr.common.util.JavaBinCodec;
+import org.apache.solr.common.util.SimpleOrderedMap;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.metrics.SolrMetricsContext;
 import org.apache.solr.request.SolrQueryRequest;
@@ -64,7 +68,9 @@
 import org.apache.solr.response.JacksonJsonWriter;
 import org.apache.solr.response.JavaBinResponseWriter;
 import org.apache.solr.response.QueryResponseWriter;
+import org.apache.solr.response.ResultContext;
 import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.response.transform.DocTransformer;
 import org.apache.solr.schema.BoolField;
 import org.apache.solr.schema.DateValueFieldType;
 import org.apache.solr.schema.DoubleValueFieldType;
@@ -76,7 +82,10 @@
 import org.apache.solr.schema.SchemaField;
 import org.apache.solr.schema.SortableTextField;
 import org.apache.solr.schema.StrField;
+import org.apache.solr.search.DocIterationInfo;
+import org.apache.solr.search.DocList;
 import org.apache.solr.search.DocValuesIteratorCache;
+import org.apache.solr.search.ReturnFields;
 import org.apache.solr.search.SolrIndexSearcher;
 import org.apache.solr.search.SolrReturnFields;
 import org.apache.solr.search.SortSpec;
@@ -124,7 +133,9 @@ public void write(SortDoc sortDoc, LeafReaderContext readerContext, EntryWriter
   private int priorityQueueSize;
   StreamExpression streamExpression;
   StreamContext streamContext;
+  SolrReturnFields returnFields;
   List<FieldWriter> fieldWriters;
+  DocTransformer transformer;
   int totalHits = 0;
   FixedBitSet[] sets = null;
   PushWriter writer;
@@ -268,35 +279,38 @@ private void _write(OutputStream os) throws IOException {
       }
     }
     SolrParams params = req.getParams();
-    String fl = params.get("fl");
 
-    String[] fields = null;
-
-    if (fl == null) {
+    if (params.get(CommonParams.FL) == null) {
       writeException(
           (new IOException(new SyntaxError("export field list (fl) must be specified."))),
           writer,
           true);
       return;
-    } else {
-      fields = fl.split(",");
-
-      for (int i = 0; i < fields.length; i++) {
+    }
 
-        fields[i] = fields[i].trim();
+    try {
+      returnFields = new SolrReturnFields(req);
+    } catch (Exception e) {
+      writeException(e, writer, true);
+      return;
+    }
 
-        if (fields[i].equals("score")) {
-          writeException(
-              (new IOException(new SyntaxError("Scoring is not currently supported with xsort."))),
-              writer,
-              true);
-          return;
-        }
-      }
+    if (returnFields.wantsScore()) {
+      writeException(
+          (new IOException(new SyntaxError("Scoring is not currently supported with xsort."))),
+          writer,
+          true);
+      return;
     }
 
     try {
-      fieldWriters = getFieldWriters(fields, req);
+      fieldWriters = getFieldWriters(returnFields, req);
+      transformer = returnFields.getTransformer();
+      if (transformer != null) {
+        transformer.setContext(
+            new ExportResultContext(
+                returnFields, searcher, info.getResponseBuilder().getQuery(), req));
+      }
     } catch (Exception e) {
       writeException(e, writer, true);
       return;
@@ -483,49 +497,89 @@ void writeDoc(
       throws IOException {
     int ord = sortDoc.ord;
     LeafReaderContext context = leaves.get(ord);
+    if (transformer == null) {
+      for (FieldWriter fieldWriter : writers) {
+        fieldWriter.write(sortDoc, context, ew);
+      }
+      return;
+    }
+
+    SolrDocument doc = new SolrDocument();
+    EntryWriter docWriter =
+        new EntryWriter() {
+          @Override
+          public EntryWriter put(CharSequence k, Object v) {
+            doc.setField(k.toString(), materialize(v));
+            return this;
+          }
+        };
     for (FieldWriter fieldWriter : writers) {
-      fieldWriter.write(sortDoc, context, ew);
+      fieldWriter.write(sortDoc, context, docWriter);
+    }
+    transformer.transform(doc, sortDoc.docBase + sortDoc.docId, DocIterationInfo.NONE);
+    for (Map.Entry<String, Object> entry : doc) {
+      if (returnFields.wantsField(entry.getKey())) {
+        ew.put(entry.getKey(), entry.getValue());
+      }
     }
   }
 
-  public List<FieldWriter> getFieldWriters(String[] fields, SolrQueryRequest req)
+  /** Converts lazily written values to plain Java objects that can outlive the write call. */
+  static Object materialize(Object v) {
+    if (v instanceof IteratorWriter iteratorWriter) {
+      List<Object> lst = new ArrayList<>();
+      iteratorWriter.toList(lst);
+      return lst;
+    } else if (v instanceof MapWriter mapWriter) {
+      return new SimpleOrderedMap<>(mapWriter);
+    }
+    return v;
+  }
+
+  List<FieldWriter> getFieldWriters(SolrReturnFields solrReturnFields, SolrQueryRequest req)
       throws IOException {
     DocValuesIteratorCache dvIterCache = new DocValuesIteratorCache(req.getSearcher(), false);
-    SolrReturnFields solrReturnFields = new SolrReturnFields(fields, req);
     boolean includeStoredFields = req.getParams().getBool(INCLUDE_STORED_FIELDS_PARAM, false);
 
+    // Also fetch fields only a transformer reads, e.g. the f= of [geo]; wantsField excludes them
+    Set<String> fieldsToFetch =
+        solrReturnFields.getLuceneFieldNames() != null
+            ? solrReturnFields.getLuceneFieldNames()
+            : Set.of();
+    // Non-exportable fields named here error; ones matched only by a glob are skipped.
+    // Includes fieldsToFetch, since requested names hold an alias, not its source field.
+    Set<String> explicitlyNamedFields = new HashSet<>(fieldsToFetch);
+    if (solrReturnFields.getRequestedFieldNames() != null) {
+      explicitlyNamedFields.addAll(solrReturnFields.getRequestedFieldNames());
+    }
+
     List<FieldWriter> writers = new ArrayList<>();
     Set<SchemaField> docValueFields = new LinkedHashSet<>();
     Map<String, SchemaField> storedFields = new LinkedHashMap<>();
 
     for (String field : req.getSearcher().getFieldNames()) {
-      if (!solrReturnFields.wantsField(field)) {
+      if (!solrReturnFields.wantsField(field) && !fieldsToFetch.contains(field)) {
         continue;
       }
       SchemaField schemaField = req.getSchema().getField(field);
       FieldType fieldType = schemaField.getType();
 
-      Set<String> requestFieldNames =
-          solrReturnFields.getRequestedFieldNames() == null
-              ? Set.of()
-              : solrReturnFields.getRequestedFieldNames();
-
       if (canUseDocValues(schemaField, fieldType)) {
         // Prefer DocValues when available
         docValueFields.add(schemaField);
       } else if (schemaField.stored()) {
         // Field is stored-only (no usable DocValues)
         if (includeStoredFields) {
           storedFields.put(field, schemaField);
-        } else if (requestFieldNames.contains(field)) {
+        } else if (explicitlyNamedFields.contains(field)) {
           // Explicitly requested field without DocValues and includeStoredFields=false
           throw new IOException(
               schemaField
                   + " must have DocValues to use this feature. "
                   + "Try setting includeStoredFields=true to retrieve this field from stored values.");
         }
         // Else: glob matched stored-only field without includeStoredFields - silently skip
-      } else if (requestFieldNames.contains(field)) {
+      } else if (explicitlyNamedFields.contains(field)) {
         // Explicitly requested field that has neither DocValues nor stored
         if (fieldType instanceof SortableTextField && !schemaField.useDocValuesAsStored()) {
           throw new IOException(
@@ -610,6 +664,47 @@ public List<FieldWriter> getFieldWriters(String[] fields, SolrQueryRequest req)
     return writers;
   }
 
+  /** Context for transformers in {@code fl}; there is no {@link DocList} when exporting. */
+  private static final class ExportResultContext extends ResultContext {
+    private final ReturnFields returnFields;
+    private final SolrIndexSearcher searcher;
+    private final Query query;
+    private final SolrQueryRequest req;
+
+    ExportResultContext(
+        ReturnFields returnFields, SolrIndexSearcher searcher, Query query, SolrQueryRequest req) {
+      this.returnFields = returnFields;
+      this.searcher = searcher;
+      this.query = query;
+      this.req = req;
+    }
+
+    @Override
+    public DocList getDocList() {
+      return null;
+    }
+
+    @Override
+    public ReturnFields getReturnFields() {
+      return returnFields;
+    }
+
+    @Override
+    public SolrIndexSearcher getSearcher() {
+      return searcher;
+    }
+
+    @Override
+    public Query getQuery() {
+      return query;
+    }
+
+    @Override
+    public SolrQueryRequest getRequest() {
+      return req;
+    }
+  }
+
   private static boolean canUseDocValues(SchemaField schemaField, FieldType fieldType) {
     return schemaField.hasDocValues()
         // Special handling for SortableTextField: unlike other field types, it requires

solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.java

@@ -18,7 +18,6 @@
 
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.BrokenBarrierException;
 import java.util.concurrent.TimeoutException;
@@ -35,9 +34,6 @@
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
 import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
 import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-import org.apache.solr.common.IteratorWriter;
-import org.apache.solr.common.MapWriter;
-import org.apache.solr.common.util.SimpleOrderedMap;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -65,14 +61,7 @@ private static final class TupleEntryWriter implements EntryWriter {
 
     @Override
     public EntryWriter put(CharSequence k, Object v) throws IOException {
-      if (v instanceof IteratorWriter) {
-        List<Object> lst = new ArrayList<>();
-        ((IteratorWriter) v).toList(lst);
-        v = lst;
-      } else if (v instanceof MapWriter) {
-        v = new SimpleOrderedMap<>((MapWriter) v);
-      }
-      tuple.put(k.toString(), v);
+      tuple.put(k.toString(), ExportWriter.materialize(v));
       return this;
     }
   }

solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java

@@ -1308,6 +1308,75 @@ public void testGlobFields() throws Exception {
         jq.contains("\"sortabledv\""));
   }
 
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testDocTransformers() throws Exception {
+    assertU(delQ("*:*"));
+    assertU(adoc("id", "1", "intdv", "10", "intdv_m", "1", "intdv_m", "2"));
+    assertU(adoc("id", "2", "intdv", "20"));
+    assertU(commit());
+    assertU(adoc("id", "3", "intdv", "30"));
+    assertU(commit());
+
+    Map<String, Object> selectDocIds = new HashMap<>();
+    String selectRsp = h.query(req("q", "*:*", "fl", "id,[docid]", "wt", "json"));
+    for (Map<String, Object> doc :
+        (List<Map<String, Object>>)
+            Utils.getObjectByPath(
+                mapper.readValue(selectRsp, Map.class), false, "/response/docs")) {
+      selectDocIds.put((String) doc.get("id"), doc.get("[docid]"));
+    }
+    assertEquals(3, selectDocIds.size());
+
+    // [docid] matches /select's, and fields are materialized (incl. multiValued)
+    List<Map<String, Object>> docs =
+        (List<Map<String, Object>>)
+            queryJsonReturnDocs(
+                reqWithPath("/export", "q", "*:*", "fl", "id,intdv_m,[docid]", "sort", "id asc"));
+    assertEquals(3, docs.size());
+    for (Map<String, Object> doc : docs) {
+      assertEquals(doc.toString(), selectDocIds.get(doc.get("id")), doc.get("[docid]"));
+    }
+    assertEquals(List.of(1, 2), docs.get(0).get("intdv_m"));
+
+    // alias renames; transformer local params may contain commas
+    docs =
+        (List<Map<String, Object>>)
+            queryJsonReturnDocs(
+                reqWithPath(
+                    "/export", "q", "*:*", "fl", "id,x:intdv,v:[value v='a,b']", "sort", "id asc"));
+    assertEquals(3, docs.size());
+    assertEquals(Map.of("id", "1", "x", 10, "v", "a,b"), docs.get(0));
+
+    // transformed fields are visible to streaming expressions
+    docs =
+        (List<Map<String, Object>>)
+            queryJsonReturnDocs(
+                reqWithPath(
+                    "/export",
+                    "q",
+                    "*:*",
+                    "fl",
+                    "id,[docid]",
+                    "sort",
+                    "id asc",
+                    "expr",
+                    "top(n=2,input(),sort=\"id desc\")"));
+    assertEquals(2, docs.size());
+    assertEquals("3", docs.get(0).get("id"));
+    assertEquals(selectDocIds.get("3"), docs.get(0).get("[docid]"));
+
+    // score is still rejected
+    docs =
+        (List<Map<String, Object>>)
+            queryJsonReturnDocs(
+                reqWithPath("/export", "q", "*:*", "fl", "id,score", "sort", "id asc"));
+    assertEquals(1, docs.size());
+    assertTrue(
+        docs.get(0).toString(),
+        docs.get(0).get(StreamParams.EXCEPTION).toString().contains("Scoring"));
+  }
+
   @SuppressWarnings("rawtypes")
   private void validateSort(int numDocs) throws Exception {
     // 10 fields

solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc

@@ -101,6 +101,10 @@ Note that sort fields still require docValues, regardless of this setting.
 
 Wildcard patterns can be used for the field list (e.g. `fl=*_i`) and will be expanded to the list of fields that match the pattern and are able to be exported, see <<Field Requirements>>.
 
+Field aliases (e.g. `fl=id,price:price_d`) and xref:document-transformers.adoc[document transformers] (e.g. `fl=id,[docid]`) are supported, and their output is also visible to a local streaming expression.
+Transformers are applied one document at a time, and those that depend on scores or on a list of matching documents won't work.
+Some transformers, like `[subquery]` and `[child]`, produce output that might not work well, or at all, with `/export` and streaming expressions.
+
 Returning scores is not supported at this time.
 
 === Specifying the Local Streaming Expression