From dd956e892937395a91225bd6d9e02837c7a8a830 Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Sat, 3 Oct 2026 23:56:13 -0400 Subject: [PATCH 01/13] SOLR-12505: fetch() streaming expression sends defType=lucene The batch query uses local params that only the lucene parser honors. Send defType=lucene so a handler default such as edismax does not defeat fetch(). Adds an unrun, hypothetical regression test. --- .../client/solrj/io/stream/FetchStream.java | 3 + .../solrj/io/stream/StreamDecoratorTest.java | 58 +++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FetchStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FetchStream.java index 4f740539f806..8074ade3e16f 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FetchStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FetchStream.java @@ -244,6 +244,9 @@ private void fetchBatch() throws IOException { ModifiableSolrParams params = new ModifiableSolrParams(); params.add("q", buf.toString()); + // The local-params prefix above is only honored by the lucene parser; do not inherit another + // defType (e.g. edismax) from the target request handler defaults. + params.add("defType", "lucene"); params.add("fl", fieldList + appendFields()); params.add("rows", Integer.toString(batchSize)); params.add(SORT, "_version_ desc"); diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java index 6337f6261894..e73e4099c0fe 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java @@ -29,6 +29,7 @@ import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.ComparatorOrder; @@ -60,6 +61,7 @@ import org.apache.solr.client.solrj.io.stream.metrics.SumMetric; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; @@ -941,6 +943,62 @@ public void testParallelHavingStream() throws Exception { solrClientCache.close(); } + /** + * SOLR-12505: fetch() sends its batch query as "{! df=.. q.op=OR}.." and must not depend on the + * target handler's default defType. NOTE: written without being compiled or run; see + * SOLR-12505-TESTING.md. + */ + @Test + public void testFetchStreamWithNonLuceneDefaultDefType() throws Exception { + final String edismaxCollection = "fetchEdismax"; + CollectionAdminRequest.createCollection(edismaxCollection, "conf", 1, 1) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(edismaxCollection, 1, 1); + try { + // make the target handler default to a parser that ignores the {! df=.. q.op=OR} local params + cluster + .getSolrClient() + .request( + new V2Request.Builder("/c/" + edismaxCollection + "/config") + .withMethod(SolrRequest.METHOD.POST) + .withPayload( + "{'update-requesthandler': {'name': '/select'," + + " 'class': 'solr.SearchHandler', 'defaults': {'defType': 'edismax'}}}") + .build()); + + new UpdateRequest() + .add(id, "0", "a_s", "hello0", "a_i", "0", "subject", "blah blah blah 0") + .add(id, "1", "a_s", "hello1", "a_i", "1", "subject", "blah blah blah 1") + .add(id, "2", "a_s", "hello2", "a_i", "2", "subject", "blah blah blah 2") + .commit(cluster.getSolrClient(), edismaxCollection); + + StreamFactory factory = + new StreamFactory() + .withCollectionUseThisConnection(edismaxCollection, getSolrConnection()) + .withFunctionName("search", CloudSolrStream.class) + .withFunctionName("fetch", FetchStream.class); + + TupleStream stream = + factory.constructStream( + "fetch(" + + edismaxCollection + + ", search(" + + edismaxCollection + + ", q=*:*, fl=\"id,a_s,a_i\", sort=\"a_i asc\"), on=\"id=a_i\"," + + " batchSize=\"2\", fl=\"subject\")"); + StreamContext context = new StreamContext(); + context.setSolrClientCache(new SolrClientCache()); + stream.setStreamContext(context); + List tuples = getTuples(stream); + + assertEquals(3, tuples.size()); + for (int i = 0; i < 3; i++) { + assertEquals("blah blah blah " + i, tuples.get(i).getString("subject")); + } + } finally { + CollectionAdminRequest.deleteCollection(edismaxCollection).process(cluster.getSolrClient()); + } + } @Test public void testFetchStream() throws Exception { From bb266a39ba04867a59a8b2fb2db1d1dd8cc357fe Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Sat, 3 Oct 2026 23:56:24 -0400 Subject: [PATCH 02/13] SOLR-12505: add hypothetical-reproduction handoff doc --- SOLR-12505-TESTING.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 SOLR-12505-TESTING.md diff --git a/SOLR-12505-TESTING.md b/SOLR-12505-TESTING.md new file mode 100644 index 000000000000..bce91fd56ac5 --- /dev/null +++ b/SOLR-12505-TESTING.md @@ -0,0 +1,39 @@ +# SOLR-12505 - hypothetical-reproduction handoff + +**Read this first: the regression test on this branch was written without being compiled or run.** The research/implement pipeline has +no Gradle access, so the test is a best-guess reproduction. Treat it as a hypothesis to confirm, not as proof. + +- JIRA: https://issues.apache.org/jira/browse/SOLR-12505 - "Streaming expressions - fetch() does not work as expected" (reported against 7.3.1, still open) +- Branch: `solr-12505-submit`, based on `apache/solr` main `14c7aac0d15` +- Commits: (1) fix + test, (2) this file (kept separate so it can be dropped before opening a PR) + +## The bug, as understood +`FetchStream` batches left-hand keys into a query of the form `{! df= q.op=OR cache=false } k1 k2 ...` and sends it to the target collection's +`/select` with no `defType`. `df`/`q.op` local params only take effect under the lucene parser. If the handler's default `defType` is something else +(reporters hit it with `defType=edismax` in `/select`; Eric Pugh confirmed in the ticket), the keys are parsed by that parser instead, nothing matches, and +`fetch()` silently returns the un-enriched input tuples (leftOuterJoin works because it uses a different path). +The ticket discussion (Joel Bernstein, David Smiley) settled on sending `defType=lucene` as a request param rather than `{!lucene ...}` local params, because +SOLR-11501 restricts parser selection through local params. + +## What the branch changes +- `FetchStream.java`: adds `params.add("defType", "lucene")` to the batch request. +- `StreamDecoratorTest.testFetchStreamWithNonLuceneDefaultDefType`: creates a 1-shard collection, rewrites `/select` via the Config API to default to `defType=edismax`, + indexes 3 docs, runs `fetch(... on="id=a_i" ...)` and asserts all 3 tuples are enriched with `subject`. + +## What was guessed (verify these first) +1. **Config API call**: the test uses `V2Request.Builder("/c//config")` with `update-requesthandler` on `/select`. If `/select` is not declared in the `conf` + configset's solrconfig (implicit only), `update-requesthandler` will fail; switch to `create-requesthandler` or use a dedicated configset. +2. **Does the test fail without the fix?** Expected: yes (tuples come back without `subject`, so `getString("subject")` returns null). Unverified. If it passes on the base + commit, edismax is not actually ignoring the local params in this configset and the premise of the ticket needs re-checking on current main. +3. Imports/formatting: `SolrRequest`, `V2Request` imports were added by hand; run spotless. +4. Config changes propagate asynchronously; the V2 config call normally waits for all replicas, but if the test is flaky add a wait. + +## How to verify (Gradle required; not run here) +``` +.\gradlew :solr:solrj-streaming:spotlessApply +.\gradlew :solr:solrj-streaming:test --tests "org.apache.solr.client.solrj.io.stream.StreamDecoratorTest.testFetchStream*" +``` +Then fail-before: revert only the `FetchStream.java` hunk and confirm the new test fails for the reason above. + +## Not done +No changelog fragment (add `changelog/unreleased/SOLR-12505-*.yml` with author `Nick Shanin` before any PR). No JIRA comment or PR was created. From 6003dd0cb01ff9f19f97c97de90ca15279a831a7 Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Sun, 4 Oct 2026 00:01:03 -0400 Subject: [PATCH 03/13] SOLR-12505: add changelog fragment --- .../unreleased/SOLR-12505-fetch-stream-deftype.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 changelog/unreleased/SOLR-12505-fetch-stream-deftype.yml diff --git a/changelog/unreleased/SOLR-12505-fetch-stream-deftype.yml b/changelog/unreleased/SOLR-12505-fetch-stream-deftype.yml new file mode 100644 index 000000000000..b034e43c96a7 --- /dev/null +++ b/changelog/unreleased/SOLR-12505-fetch-stream-deftype.yml @@ -0,0 +1,10 @@ +title: > + The fetch() streaming expression now sends defType=lucene with its batch query, so it no longer + fails to enrich tuples when the target request handler defaults to a different query parser. +type: fixed +authors: + - name: Nick Shanin + nick: nick-boss-tech +links: + - name: SOLR-12505 + url: https://issues.apache.org/jira/browse/SOLR-12505 From 7dbb80dd1f18fbc02d9b1b87afef9a020c999fa3 Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Sun, 4 Oct 2026 23:11:31 +0000 Subject: [PATCH 04/13] SOLR-12505: fix the changelog author entry --- changelog/unreleased/SOLR-12505-fetch-stream-deftype.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/changelog/unreleased/SOLR-12505-fetch-stream-deftype.yml b/changelog/unreleased/SOLR-12505-fetch-stream-deftype.yml index b034e43c96a7..8395af3fbdf4 100644 --- a/changelog/unreleased/SOLR-12505-fetch-stream-deftype.yml +++ b/changelog/unreleased/SOLR-12505-fetch-stream-deftype.yml @@ -4,7 +4,6 @@ title: > type: fixed authors: - name: Nick Shanin - nick: nick-boss-tech links: - name: SOLR-12505 url: https://issues.apache.org/jira/browse/SOLR-12505 From 7b3656af6fa6904d3c4c9b52cfe06aba1e25642f Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Mon, 5 Oct 2026 01:22:51 +0000 Subject: [PATCH 05/13] SOLR-12505: make the fetch defType test discriminate The subjects indexed by testFetchStreamWithNonLuceneDefaultDefType ended in the join keys (0, 1, 2) and are copyFielded into text, the handler default field, so the default edismax parser found the documents by accident and the test passed without the fix. The subjects are now alpha, bravo and charlie, which share no token with the keys; on base code the test fails with the fetched subject null, the ticket symptom. Also drops the stale javadoc note claiming the test was never run. --- SOLR-12505-TESTING.md | 39 ------------------- .../solrj/io/stream/StreamDecoratorTest.java | 16 +++++--- 2 files changed, 10 insertions(+), 45 deletions(-) delete mode 100644 SOLR-12505-TESTING.md diff --git a/SOLR-12505-TESTING.md b/SOLR-12505-TESTING.md deleted file mode 100644 index bce91fd56ac5..000000000000 --- a/SOLR-12505-TESTING.md +++ /dev/null @@ -1,39 +0,0 @@ -# SOLR-12505 - hypothetical-reproduction handoff - -**Read this first: the regression test on this branch was written without being compiled or run.** The research/implement pipeline has -no Gradle access, so the test is a best-guess reproduction. Treat it as a hypothesis to confirm, not as proof. - -- JIRA: https://issues.apache.org/jira/browse/SOLR-12505 - "Streaming expressions - fetch() does not work as expected" (reported against 7.3.1, still open) -- Branch: `solr-12505-submit`, based on `apache/solr` main `14c7aac0d15` -- Commits: (1) fix + test, (2) this file (kept separate so it can be dropped before opening a PR) - -## The bug, as understood -`FetchStream` batches left-hand keys into a query of the form `{! df= q.op=OR cache=false } k1 k2 ...` and sends it to the target collection's -`/select` with no `defType`. `df`/`q.op` local params only take effect under the lucene parser. If the handler's default `defType` is something else -(reporters hit it with `defType=edismax` in `/select`; Eric Pugh confirmed in the ticket), the keys are parsed by that parser instead, nothing matches, and -`fetch()` silently returns the un-enriched input tuples (leftOuterJoin works because it uses a different path). -The ticket discussion (Joel Bernstein, David Smiley) settled on sending `defType=lucene` as a request param rather than `{!lucene ...}` local params, because -SOLR-11501 restricts parser selection through local params. - -## What the branch changes -- `FetchStream.java`: adds `params.add("defType", "lucene")` to the batch request. -- `StreamDecoratorTest.testFetchStreamWithNonLuceneDefaultDefType`: creates a 1-shard collection, rewrites `/select` via the Config API to default to `defType=edismax`, - indexes 3 docs, runs `fetch(... on="id=a_i" ...)` and asserts all 3 tuples are enriched with `subject`. - -## What was guessed (verify these first) -1. **Config API call**: the test uses `V2Request.Builder("/c//config")` with `update-requesthandler` on `/select`. If `/select` is not declared in the `conf` - configset's solrconfig (implicit only), `update-requesthandler` will fail; switch to `create-requesthandler` or use a dedicated configset. -2. **Does the test fail without the fix?** Expected: yes (tuples come back without `subject`, so `getString("subject")` returns null). Unverified. If it passes on the base - commit, edismax is not actually ignoring the local params in this configset and the premise of the ticket needs re-checking on current main. -3. Imports/formatting: `SolrRequest`, `V2Request` imports were added by hand; run spotless. -4. Config changes propagate asynchronously; the V2 config call normally waits for all replicas, but if the test is flaky add a wait. - -## How to verify (Gradle required; not run here) -``` -.\gradlew :solr:solrj-streaming:spotlessApply -.\gradlew :solr:solrj-streaming:test --tests "org.apache.solr.client.solrj.io.stream.StreamDecoratorTest.testFetchStream*" -``` -Then fail-before: revert only the `FetchStream.java` hunk and confirm the new test fails for the reason above. - -## Not done -No changelog fragment (add `changelog/unreleased/SOLR-12505-*.yml` with author `Nick Shanin` before any PR). No JIRA comment or PR was created. diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java index e73e4099c0fe..473d197ac3e2 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java @@ -945,8 +945,7 @@ public void testParallelHavingStream() throws Exception { /** * SOLR-12505: fetch() sends its batch query as "{! df=.. q.op=OR}.." and must not depend on the - * target handler's default defType. NOTE: written without being compiled or run; see - * SOLR-12505-TESTING.md. + * target handler's default defType. */ @Test public void testFetchStreamWithNonLuceneDefaultDefType() throws Exception { @@ -966,10 +965,13 @@ public void testFetchStreamWithNonLuceneDefaultDefType() throws Exception { + " 'class': 'solr.SearchHandler', 'defaults': {'defType': 'edismax'}}}") .build()); + // The subject values deliberately share no token with the join keys (0, 1, 2): the schema + // copyFields subject into text, the handler default field, so a token in common would let + // the default edismax parser find the documents by accident and hide the defType bug. new UpdateRequest() - .add(id, "0", "a_s", "hello0", "a_i", "0", "subject", "blah blah blah 0") - .add(id, "1", "a_s", "hello1", "a_i", "1", "subject", "blah blah blah 1") - .add(id, "2", "a_s", "hello2", "a_i", "2", "subject", "blah blah blah 2") + .add(id, "0", "a_s", "hello0", "a_i", "0", "subject", "alpha") + .add(id, "1", "a_s", "hello1", "a_i", "1", "subject", "bravo") + .add(id, "2", "a_s", "hello2", "a_i", "2", "subject", "charlie") .commit(cluster.getSolrClient(), edismaxCollection); StreamFactory factory = @@ -992,13 +994,15 @@ public void testFetchStreamWithNonLuceneDefaultDefType() throws Exception { List tuples = getTuples(stream); assertEquals(3, tuples.size()); + final String[] expectedSubjects = {"alpha", "bravo", "charlie"}; for (int i = 0; i < 3; i++) { - assertEquals("blah blah blah " + i, tuples.get(i).getString("subject")); + assertEquals(expectedSubjects[i], tuples.get(i).getString("subject")); } } finally { CollectionAdminRequest.deleteCollection(edismaxCollection).process(cluster.getSolrClient()); } } + @Test public void testFetchStream() throws Exception { From f9abd671915c9fa994a58ebc60a51fab2b97eb9a Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Mon, 5 Oct 2026 01:53:47 +0000 Subject: [PATCH 06/13] SOLR-12505: close the SolrClientCache in the fetch defType test The new test built its StreamContext with a fresh SolrClientCache and never closed it, leaking the cloud client at class teardown. The cache is now closed in the finally block. --- .../solr/client/solrj/io/stream/StreamDecoratorTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java index 473d197ac3e2..445e35b87cab 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java @@ -953,6 +953,7 @@ public void testFetchStreamWithNonLuceneDefaultDefType() throws Exception { CollectionAdminRequest.createCollection(edismaxCollection, "conf", 1, 1) .process(cluster.getSolrClient()); cluster.waitForActiveCollection(edismaxCollection, 1, 1); + SolrClientCache solrClientCache = new SolrClientCache(); try { // make the target handler default to a parser that ignores the {! df=.. q.op=OR} local params cluster @@ -989,7 +990,7 @@ public void testFetchStreamWithNonLuceneDefaultDefType() throws Exception { + ", q=*:*, fl=\"id,a_s,a_i\", sort=\"a_i asc\"), on=\"id=a_i\"," + " batchSize=\"2\", fl=\"subject\")"); StreamContext context = new StreamContext(); - context.setSolrClientCache(new SolrClientCache()); + context.setSolrClientCache(solrClientCache); stream.setStreamContext(context); List tuples = getTuples(stream); @@ -999,6 +1000,7 @@ public void testFetchStreamWithNonLuceneDefaultDefType() throws Exception { assertEquals(expectedSubjects[i], tuples.get(i).getString("subject")); } } finally { + solrClientCache.close(); CollectionAdminRequest.deleteCollection(edismaxCollection).process(cluster.getSolrClient()); } } From a03ea60ad0a94d5c5e777081b16cb36ebc4c3590 Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Mon, 12 Oct 2026 20:41:07 +0000 Subject: [PATCH 07/13] SOLR-12505: reject a fetch join field name that would break the batch query --- .../client/solrj/io/stream/FetchStream.java | 8 +++++ .../solrj/io/stream/StreamDecoratorTest.java | 33 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FetchStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FetchStream.java index 8074ade3e16f..77a5812426d9 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FetchStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FetchStream.java @@ -150,6 +150,14 @@ private void init( leftKey = rightKey = on; } + if (rightKey.matches(".*[\\s{}!'\"\\\\].*")) { + throw new IOException( + String.format( + Locale.ROOT, + "fetch on parameter has an invalid join field name [%s]: whitespace, braces, '!', quotes and backslashes are not allowed", + rightKey)); + } + for (int i = 0; i < fields.length; i++) { fields[i] = fields[i].trim(); if (fields[i].equals(VERSION_FIELD)) { diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java index 445e35b87cab..a4037413a114 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java @@ -1005,6 +1005,39 @@ public void testFetchStreamWithNonLuceneDefaultDefType() throws Exception { } } + @Test + public void testFetchStreamRejectsJoinFieldNameThatBreaksLocalParams() throws Exception { + SolrClientCache solrClientCache = new SolrClientCache(); + try { + StreamFactory factory = + new StreamFactory() + .withCollectionUseThisConnection(COLLECTIONORALIAS, getSolrConnection()) + .withFunctionName("search", CloudSolrStream.class) + .withFunctionName("fetch", FetchStream.class); + + IOException e = + assertThrows( + IOException.class, + () -> { + TupleStream stream = + factory.constructStream( + "fetch(" + + COLLECTIONORALIAS + + ", search(" + + COLLECTIONORALIAS + + ", q=*:*, fl=\"id,a_s,a_i\", sort=\"a_i asc\"), on=\"id=a i\"," + + " batchSize=\"2\", fl=\"subject\")"); + StreamContext context = new StreamContext(); + context.setSolrClientCache(solrClientCache); + stream.setStreamContext(context); + getTuples(stream); + }); + assertTrue(e.getMessage(), e.getMessage().contains("invalid join field name [a i]")); + } finally { + solrClientCache.close(); + } + } + @Test public void testFetchStream() throws Exception { From 9b81c1fa29727c81c09b58c98f71e70064b6fb86 Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Mon, 12 Oct 2026 23:03:39 +0000 Subject: [PATCH 08/13] SOLR-12505: check the fetch join field rejection on the cause chain and keep assertThrows lambdas to one statement --- .../solrj/io/stream/StreamDecoratorTest.java | 44 ++++++++++++------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java index a4037413a114..6935afb9a391 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java @@ -1015,29 +1015,41 @@ public void testFetchStreamRejectsJoinFieldNameThatBreaksLocalParams() throws Ex .withFunctionName("search", CloudSolrStream.class) .withFunctionName("fetch", FetchStream.class); + String expression = + "fetch(" + + COLLECTIONORALIAS + + ", search(" + + COLLECTIONORALIAS + + ", q=*:*, fl=\"id,a_s,a_i\", sort=\"a_i asc\"), on=\"id=a i\"," + + " batchSize=\"2\", fl=\"subject\")"; IOException e = assertThrows( - IOException.class, - () -> { - TupleStream stream = - factory.constructStream( - "fetch(" - + COLLECTIONORALIAS - + ", search(" - + COLLECTIONORALIAS - + ", q=*:*, fl=\"id,a_s,a_i\", sort=\"a_i asc\"), on=\"id=a i\"," - + " batchSize=\"2\", fl=\"subject\")"); - StreamContext context = new StreamContext(); - context.setSolrClientCache(solrClientCache); - stream.setStreamContext(context); - getTuples(stream); - }); - assertTrue(e.getMessage(), e.getMessage().contains("invalid join field name [a i]")); + IOException.class, () -> drainFetchStream(factory, solrClientCache, expression)); + assertTrue(e.getMessage(), causeChainContains(e, "invalid join field name [a i]")); } finally { solrClientCache.close(); } } + private List drainFetchStream( + StreamFactory factory, SolrClientCache solrClientCache, String expression) + throws IOException { + TupleStream stream = factory.constructStream(expression); + StreamContext context = new StreamContext(); + context.setSolrClientCache(solrClientCache); + stream.setStreamContext(context); + return getTuples(stream); + } + + private static boolean causeChainContains(Throwable t, String text) { + for (; t != null; t = t.getCause()) { + if (t.getMessage() != null && t.getMessage().contains(text)) { + return true; + } + } + return false; + } + @Test public void testFetchStream() throws Exception { From e9571be71bc4f71bd41e2e8017aafd4c1686c5dd Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Tue, 13 Oct 2026 16:29:05 +0000 Subject: [PATCH 09/13] SOLR-12505: seed documents in the join field rejection test --- .../solr/client/solrj/io/stream/StreamDecoratorTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java index 6935afb9a391..b4abc5b8f504 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java @@ -1008,6 +1008,10 @@ public void testFetchStreamWithNonLuceneDefaultDefType() throws Exception { @Test public void testFetchStreamRejectsJoinFieldNameThatBreaksLocalParams() throws Exception { SolrClientCache solrClientCache = new SolrClientCache(); + new UpdateRequest() + .add(id, "0", "a_s", "hello0", "a_i", "0") + .add(id, "1", "a_s", "hello1", "a_i", "1") + .commit(cluster.getSolrClient(), COLLECTIONORALIAS); try { StreamFactory factory = new StreamFactory() From efc9126b417a3e357d1be49d31c65eb6aecec6fc Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Wed, 14 Oct 2026 23:31:06 +0000 Subject: [PATCH 10/13] SOLR-12505: widen the fetch join field name check to Character.isWhitespace and name the check in the changelog title --- .../SOLR-12505-fetch-stream-deftype.yml | 3 +- .../client/solrj/io/stream/FetchStream.java | 21 +++++++++++- .../solrj/io/stream/StreamDecoratorTest.java | 33 +++++++++++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/changelog/unreleased/SOLR-12505-fetch-stream-deftype.yml b/changelog/unreleased/SOLR-12505-fetch-stream-deftype.yml index 8395af3fbdf4..36ecebc53daf 100644 --- a/changelog/unreleased/SOLR-12505-fetch-stream-deftype.yml +++ b/changelog/unreleased/SOLR-12505-fetch-stream-deftype.yml @@ -1,6 +1,5 @@ title: > - The fetch() streaming expression now sends defType=lucene with its batch query, so it no longer - fails to enrich tuples when the target request handler defaults to a different query parser. + The fetch() streaming expression now sends defType=lucene with its batch query, so it no longer fails to enrich tuples when the target request handler defaults to a different query parser, and it rejects a join field name that would break that query. type: fixed authors: - name: Nick Shanin diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FetchStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FetchStream.java index 77a5812426d9..891b0f465fa2 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FetchStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FetchStream.java @@ -150,7 +150,7 @@ private void init( leftKey = rightKey = on; } - if (rightKey.matches(".*[\\s{}!'\"\\\\].*")) { + if (containsBatchBreakingChar(rightKey)) { throw new IOException( String.format( Locale.ROOT, @@ -170,6 +170,25 @@ private void init( } } + // The batch query embeds the join field name in local params, whose reader splits on + // Character.isWhitespace; reject anything that reader would treat as a separator, plus the + // other characters that would end or corrupt the df= value. + private static boolean containsBatchBreakingChar(String name) { + for (int i = 0; i < name.length(); i++) { + char c = name.charAt(i); + if (Character.isWhitespace(c) + || c == '{' + || c == '}' + || c == '!' + || c == '\'' + || c == '"' + || c == '\\') { + return true; + } + } + return false; + } + @Override public StreamExpression toExpression(StreamFactory factory) throws IOException { return toExpression(factory, true); diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java index b4abc5b8f504..a83fed363d30 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java @@ -1035,6 +1035,39 @@ public void testFetchStreamRejectsJoinFieldNameThatBreaksLocalParams() throws Ex } } + @Test + public void testFetchStreamRejectsJoinFieldNameWithUnicodeSpace() throws Exception { + SolrClientCache solrClientCache = new SolrClientCache(); + new UpdateRequest() + .add(id, "0", "a_s", "hello0", "a_i", "0") + .add(id, "1", "a_s", "hello1", "a_i", "1") + .commit(cluster.getSolrClient(), COLLECTIONORALIAS); + try { + StreamFactory factory = + new StreamFactory() + .withCollectionUseThisConnection(COLLECTIONORALIAS, getSolrConnection()) + .withFunctionName("search", CloudSolrStream.class) + .withFunctionName("fetch", FetchStream.class); + + // The join field name carries an em space (U+2003). The local-params reader splits on + // Character.isWhitespace, so this name breaks the batch query the same way "a i" does, + // even though it has no ASCII whitespace. + String expression = + "fetch(" + + COLLECTIONORALIAS + + ", search(" + + COLLECTIONORALIAS + + ", q=*:*, fl=\"id,a_s,a_i\", sort=\"a_i asc\"), on=\"id=a\u2003i\"," + + " batchSize=\"2\", fl=\"subject\")"; + IOException e = + assertThrows( + IOException.class, () -> drainFetchStream(factory, solrClientCache, expression)); + assertTrue(e.getMessage(), causeChainContains(e, "invalid join field name [a\u2003i]")); + } finally { + solrClientCache.close(); + } + } + private List drainFetchStream( StreamFactory factory, SolrClientCache solrClientCache, String expression) throws IOException { From c7f10e5b18bbb57c4d3bb1b3fa030fbf70967185 Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Fri, 16 Oct 2026 07:44:43 +0000 Subject: [PATCH 11/13] SOLR-12505: narrow the fetch join field name check to the shapes that break the local-params value, and reject a leading '$' --- .../client/solrj/io/stream/FetchStream.java | 19 +++++------ .../solrj/io/stream/StreamDecoratorTest.java | 32 +++++++++++++++++++ 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FetchStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FetchStream.java index 891b0f465fa2..b39629146e44 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FetchStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FetchStream.java @@ -154,7 +154,7 @@ private void init( throw new IOException( String.format( Locale.ROOT, - "fetch on parameter has an invalid join field name [%s]: whitespace, braces, '!', quotes and backslashes are not allowed", + "fetch on parameter has an invalid join field name [%s]: whitespace, '}', a leading quote and a leading '$' are not allowed", rightKey)); } @@ -170,19 +170,16 @@ private void init( } } - // The batch query embeds the join field name in local params, whose reader splits on - // Character.isWhitespace; reject anything that reader would treat as a separator, plus the - // other characters that would end or corrupt the df= value. + // The batch query embeds the join field name as an unquoted local-params value. That reader + // ends an unquoted literal at whitespace or '}', a leading quote starts a quoted string, and + // a leading $ dereferences a request parameter, so reject exactly those shapes. private static boolean containsBatchBreakingChar(String name) { for (int i = 0; i < name.length(); i++) { char c = name.charAt(i); - if (Character.isWhitespace(c) - || c == '{' - || c == '}' - || c == '!' - || c == '\'' - || c == '"' - || c == '\\') { + if (Character.isWhitespace(c) || c == '}') { + return true; + } + if (i == 0 && (c == '\'' || c == '"' || c == '$')) { return true; } } diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java index a83fed363d30..8bca655aaeb7 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java @@ -1068,6 +1068,38 @@ public void testFetchStreamRejectsJoinFieldNameWithUnicodeSpace() throws Excepti } } + @Test + public void testFetchStreamRejectsJoinFieldNameStartingWithDollarSign() throws Exception { + SolrClientCache solrClientCache = new SolrClientCache(); + new UpdateRequest() + .add(id, "0", "a_s", "hello0", "a_i", "0") + .add(id, "1", "a_s", "hello1", "a_i", "1") + .commit(cluster.getSolrClient(), COLLECTIONORALIAS); + try { + StreamFactory factory = + new StreamFactory() + .withCollectionUseThisConnection(COLLECTIONORALIAS, getSolrConnection()) + .withFunctionName("search", CloudSolrStream.class) + .withFunctionName("fetch", FetchStream.class); + + // A leading $ in the df value is read as a request parameter dereference, so the batch + // would look up the field named by that parameter instead of the join field itself. + String expression = + "fetch(" + + COLLECTIONORALIAS + + ", search(" + + COLLECTIONORALIAS + + ", q=*:*, fl=\"id,a_s,a_i\", sort=\"a_i asc\"), on=\"id=$a_i\"," + + " batchSize=\"2\", fl=\"subject\")"; + IOException e = + assertThrows( + IOException.class, () -> drainFetchStream(factory, solrClientCache, expression)); + assertTrue(e.getMessage(), causeChainContains(e, "invalid join field name [$a_i]")); + } finally { + solrClientCache.close(); + } + } + private List drainFetchStream( StreamFactory factory, SolrClientCache solrClientCache, String expression) throws IOException { From 6b551865e3bdb461249b948c23ac586d056301b8 Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Fri, 16 Oct 2026 16:02:51 -0600 Subject: [PATCH 12/13] SOLR-12505: test the remaining fetch join field name rejections and isolate the edismax test's Config API overlay in a dedicated configset --- .../solrj/io/stream/StreamDecoratorTest.java | 78 ++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java index 8bca655aaeb7..d777f65c7b4b 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java @@ -95,6 +95,13 @@ public static void setupCluster() throws Exception { .resolve("configsets") .resolve("streaming") .resolve("conf")) + .addConfig( + "conf-edismax", + getFile("solrj") + .resolve("solr") + .resolve("configsets") + .resolve("streaming") + .resolve("conf")) .addConfig( "ml", getFile("solrj").resolve("solr").resolve("configsets").resolve("ml").resolve("conf")) @@ -950,7 +957,10 @@ public void testParallelHavingStream() throws Exception { @Test public void testFetchStreamWithNonLuceneDefaultDefType() throws Exception { final String edismaxCollection = "fetchEdismax"; - CollectionAdminRequest.createCollection(edismaxCollection, "conf", 1, 1) + // The Config API overlay is stored with the configset, not the collection, so the /select + // overlay set below would outlive this collection and leak into every other test in this + // class that uses the shared "conf" configset. This collection uses its own copy instead. + CollectionAdminRequest.createCollection(edismaxCollection, "conf-edismax", 1, 1) .process(cluster.getSolrClient()); cluster.waitForActiveCollection(edismaxCollection, 1, 1); SolrClientCache solrClientCache = new SolrClientCache(); @@ -1100,6 +1110,72 @@ public void testFetchStreamRejectsJoinFieldNameStartingWithDollarSign() throws E } } + @Test + public void testFetchStreamRejectsJoinFieldNameWithClosingBraceOrLeadingQuote() throws Exception { + SolrClientCache solrClientCache = new SolrClientCache(); + new UpdateRequest() + .add(id, "0", "a_s", "hello0", "a_i", "0") + .add(id, "1", "a_s", "hello1", "a_i", "1") + .commit(cluster.getSolrClient(), COLLECTIONORALIAS); + try { + StreamFactory factory = + new StreamFactory() + .withCollectionUseThisConnection(COLLECTIONORALIAS, getSolrConnection()) + .withFunctionName("search", CloudSolrStream.class) + .withFunctionName("fetch", FetchStream.class); + + // A } in the name ends the unquoted local-params value early, so the rest of the name + // lands outside the local params and changes the batch query. + String braceExpression = + "fetch(" + + COLLECTIONORALIAS + + ", search(" + + COLLECTIONORALIAS + + ", q=*:*, fl=\"id,a_s,a_i\", sort=\"a_i asc\"), on=\"id=a}i\"," + + " batchSize=\"2\", fl=\"subject\")"; + IOException braceFailure = + assertThrows( + IOException.class, () -> drainFetchStream(factory, solrClientCache, braceExpression)); + assertTrue( + braceFailure.getMessage(), + causeChainContains(braceFailure, "invalid join field name [a}i]")); + + // A leading quote makes the local-params reader treat the rest of the name as a quoted + // string instead of the field name itself. + String singleQuoteExpression = + "fetch(" + + COLLECTIONORALIAS + + ", search(" + + COLLECTIONORALIAS + + ", q=*:*, fl=\"id,a_s,a_i\", sort=\"a_i asc\"), on=\"id='a_i\"," + + " batchSize=\"2\", fl=\"subject\")"; + IOException singleQuoteFailure = + assertThrows( + IOException.class, + () -> drainFetchStream(factory, solrClientCache, singleQuoteExpression)); + assertTrue( + singleQuoteFailure.getMessage(), + causeChainContains(singleQuoteFailure, "invalid join field name ['a_i]")); + + String doubleQuoteExpression = + "fetch(" + + COLLECTIONORALIAS + + ", search(" + + COLLECTIONORALIAS + + ", q=*:*, fl=\"id,a_s,a_i\", sort=\"a_i asc\"), on=\"id=\\\"a_i\"," + + " batchSize=\"2\", fl=\"subject\")"; + IOException doubleQuoteFailure = + assertThrows( + IOException.class, + () -> drainFetchStream(factory, solrClientCache, doubleQuoteExpression)); + assertTrue( + doubleQuoteFailure.getMessage(), + causeChainContains(doubleQuoteFailure, "invalid join field name [\"a_i]")); + } finally { + solrClientCache.close(); + } + } + private List drainFetchStream( StreamFactory factory, SolrClientCache solrClientCache, String expression) throws IOException { From 6677cc5cf10a7690335ab69747ba7aa78f833aef Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Fri, 23 Oct 2026 01:36:26 -0400 Subject: [PATCH 13/13] SOLR-12505: document fetch parser and join field rules --- .../modules/query-guide/pages/stream-decorator-reference.adoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/solr/solr-ref-guide/modules/query-guide/pages/stream-decorator-reference.adoc b/solr/solr-ref-guide/modules/query-guide/pages/stream-decorator-reference.adoc index 120f871f5f60..e3a3ec3fb0fc 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/stream-decorator-reference.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/stream-decorator-reference.adoc @@ -768,6 +768,7 @@ This is the approach for executing batches of streaming expressions from a `topi The `fetch` function iterates a stream and fetches additional fields and adds them to the tuples. The `fetch` function fetches in batches to limit the number of calls back to Solr. Tuples streamed from the `fetch` function will contain the original fields and the additional fields that were fetched. +The batch lookup uses `defType=lucene`, even if the target collection's `/select` handler defaults to a different query parser. The `fetch` function supports one-to-one fetches. Many-to-one fetches, where the stream source contains duplicate keys, will also work, but one-to-many fetches are currently not supported by this function. @@ -778,6 +779,7 @@ Many-to-one fetches, where the stream source contains duplicate keys, will also * `fl`: (Mandatory) The fields to be fetched. * `on`: Fields to be used for checking equality of tuples between stream source and fetched records. Formatted as `on="fieldNameInTuple=fieldNameInCollection"`. +Because the collection-side field name is inserted into a local-parameters query, it cannot contain whitespace or `}`, or start with `'`, `"`, or `$`. * `batchSize`: (Optional) The batch fetch size. === fetch Syntax