feat(java): add token-level readUtf8/writeUtf8 operations to ForyJson

docs/json/integration.md

@@ -88,3 +88,48 @@ arguments. `AbstractGenericHttpMessageConverter` preserves Java generics, but
 `TypeRef.of(javaType)` does not restore Kotlin nullability. Neither a Java `Type` nor the runtime
 value alone can recover the full Kotlin declaration. Use `jsonTypeRef(kType)` after obtaining that
 declaration; this API does not automatically install a Spring converter.
+
+## Token-level reading and writing
+
+Frameworks that bring their own (de)serializers, for example generated at compile time, can use
+Fory JSON's UTF-8 reader and writer directly instead of its object mapping. `readUtf8` and
+`writeUtf8` lend a pooled reader or writer to an operation, using the `ForyJson` instance's limits,
+caches and buffers:
+
+```java
+ForyJson json = ForyJson.builder().build();
+
+byte[] bytes =
+    json.writeUtf8(
+        writer -> {
+          writer.writeObjectStart();
+          writer.writeFieldName("id");
+          writer.writeInt(7);
+          writer.writeObjectEnd();
+        });
+
+int id =
+    json.readUtf8(
+        bytes,
+        0,
+        bytes.length,
+        reader -> {
+          reader.expectNextToken('{');
+          String name = reader.readFieldName(); // "id"
+          reader.expect(':');
+          int value = reader.readInt();
+          reader.expectNextToken('}');
+          return value;
+        });
+```
+
+The reader and writer are only valid inside the operation. `readUtf8` rejects trailing
+non-whitespace content after the operation returns.
+
+Token methods such as `expectNextToken` do not track nesting or allocation on their own. Like a
+custom codec, an external deserializer that reads untrusted input should call
+`reader.enterDepth()` / `reader.exitDepth()` around each object or array and
+`reader.reserveGraphMemory(bytes)` before allocating containers, so the instance's `maxDepth` and
+graph-memory limits apply; see [Security](security.md). Checked exceptions thrown by the operation are
+propagated unchanged. Like the other root APIs, these methods are not reentrant on the same
+`ForyJson` instance.

java/fory-json/src/main/java/org/apache/fory/json/ForyJson.java

@@ -796,6 +796,76 @@ public <T> T fromJson(byte[] bytes, int offset, int length, TypeRef<T> typeRef)
     }
   }
 
+  /**
+   * Lends a pooled {@link Utf8JsonReader} positioned at {@code bytes[offset, offset + length)} to
+   * {@code operation}, then rejects trailing non-whitespace content.
+   *
+   * <p>This is the entry point for callers that read JSON tokens directly, for example frameworks
+   * with their own deserializers. The reader shares this instance's field-name caches, buffers and
+   * resolver and must not escape the operation. Token methods do not track nesting or allocation on
+   * their own: callers reading untrusted input should use {@link Utf8JsonReader#enterDepth()},
+   * {@link Utf8JsonReader#exitDepth()} and {@link Utf8JsonReader#reserveGraphMemory(long)} as
+   * custom codecs do, so this instance's depth and graph-memory limits apply. Like the other root
+   * APIs, this method is not reentrant on the same instance: the operation must not invoke a {@code
+   * ForyJson} root API.
+   */
+  public <R, E extends Throwable> R readUtf8(
+      byte[] bytes, int offset, int length, Utf8ReadOperation<R, E> operation) throws E {
+    checkByteRange(bytes, offset, length);
+    Objects.requireNonNull(operation, "operation");
+    PooledState entry = acquire();
+    JsonState state = entry.state;
+    try {
+      state.typeResolver.lockJIT();
+      try {
+        Utf8JsonReader reader = state.utf8Reader(bytes, offset, length);
+        R value = operation.read(reader);
+        reader.finish();
+        return value;
+      } finally {
+        state.typeResolver.unlockJIT();
+      }
+    } finally {
+      try {
+        state.clearUtf8Reader();
+      } finally {
+        release(entry);
+      }
+    }
+  }
+
+  /**
+   * Lends a pooled {@link Utf8JsonWriter} to {@code operation} and returns the written UTF-8 JSON
+   * as a detached byte array.
+   *
+   * <p>This is the entry point for callers that write JSON tokens directly, for example frameworks
+   * with their own serializers. The writer shares this instance's options and resolver; it must not
+   * escape the operation. Like the other root APIs, this method is not reentrant on the same
+   * instance: the operation must not invoke a {@code ForyJson} root API.
+   */
+  public <E extends Throwable> byte[] writeUtf8(Utf8WriteOperation<E> operation) throws E {
+    Objects.requireNonNull(operation, "operation");
+    PooledState entry = acquire();
+    JsonState state = entry.state;
+    Utf8JsonWriter writer = state.utf8Writer();
+    try {
+      state.typeResolver.lockJIT();
+      try {
+        operation.write(writer);
+      } finally {
+        state.typeResolver.unlockJIT();
+      }
+      return writer.toJsonBytes();
+    } finally {
+      try {
+        state.buffer = writer.getBuffer();
+        writer.clear();
+      } finally {
+        release(entry);
+      }
+    }
+  }
+
   private static byte[] readAllBytes(InputStream input) {
     Objects.requireNonNull(input, "input");
     try {

java/fory-json/src/main/java/org/apache/fory/json/Utf8ReadOperation.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fory.json;
+
+import org.apache.fory.json.reader.Utf8JsonReader;
+
+/**
+ * Reads one JSON value with a {@link Utf8JsonReader} lent by {@link ForyJson#readUtf8(byte[], int,
+ * int, Utf8ReadOperation)}.
+ *
+ * @param <R> the result type
+ * @param <E> the checked exception the operation may throw
+ */
+@FunctionalInterface
+public interface Utf8ReadOperation<R, E extends Throwable> {
+  /**
+   * Consumes exactly one JSON value from {@code reader}. The reader is only valid during this call.
+   */
+  R read(Utf8JsonReader reader) throws E;
+}

java/fory-json/src/main/java/org/apache/fory/json/Utf8WriteOperation.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fory.json;
+
+import org.apache.fory.json.writer.Utf8JsonWriter;
+
+/**
+ * Writes one JSON value with a {@link Utf8JsonWriter} lent by {@link
+ * ForyJson#writeUtf8(Utf8WriteOperation)}.
+ *
+ * @param <E> the checked exception the operation may throw
+ */
+@FunctionalInterface
+public interface Utf8WriteOperation<E extends Throwable> {
+  /** Writes exactly one JSON value to {@code writer}. The writer is only valid during this call. */
+  void write(Utf8JsonWriter writer) throws E;
+}

java/fory-json/src/test/java/org/apache/fory/json/ForyJsonTokenOperationTest.java

@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fory.json;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertSame;
+import static org.testng.Assert.assertThrows;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.expectThrows;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import org.apache.fory.json.reader.Utf8JsonReader;
+import org.testng.annotations.Test;
+
+/** Covers the token-level reader and writer operations used by external serializers. */
+public class ForyJsonTokenOperationTest {
+  // One pooled state: an unreleased state would make the next operation wait forever.
+  private final ForyJson json =
+      ForyJson.builder().withAsyncCompilation(false).withConcurrencyLevel(1).build();
+
+  @Test
+  public void writeAndReadObjectTokens() throws IOException {
+    byte[] bytes =
+        json.writeUtf8(
+            writer -> {
+              writer.writeObjectStart();
+              writer.writeFieldName("id");
+              writer.writeInt(7);
+              writer.writeComma(1);
+              writer.writeFieldName("name");
+              writer.writeString("fory");
+              writer.writeObjectEnd();
+            });
+    assertEquals(new String(bytes, StandardCharsets.UTF_8), "{\"id\":7,\"name\":\"fory\"}");
+
+    Pair pair = json.readUtf8(bytes, 0, bytes.length, ForyJsonTokenOperationTest::readPair);
+    assertEquals(pair.id, 7);
+    assertEquals(pair.name, "fory");
+    // The same document still reads through the object-mapping root API.
+    assertEquals(json.fromJson(bytes, Pair.class).name, "fory");
+  }
+
+  @Test
+  public void readsByteRangeAndRejectsTrailingContent() {
+    byte[] bytes = "xx{\"id\":3,\"name\":\"a\"}yy".getBytes(StandardCharsets.UTF_8);
+    assertEquals(
+        json.readUtf8(bytes, 2, bytes.length - 4, ForyJsonTokenOperationTest::readPair).id, 3);
+
+    byte[] trailing = "1 2".getBytes(StandardCharsets.UTF_8);
+    assertThrows(
+        ForyJsonException.class,
+        () -> json.readUtf8(trailing, 0, trailing.length, Utf8JsonReader::readInt));
+    assertThrows(
+        IndexOutOfBoundsException.class,
+        () -> json.readUtf8(trailing, 2, 5, Utf8JsonReader::readInt));
+  }
+
+  @Test
+  public void propagatesOperationFailuresAndReleasesState() {
+    byte[] bytes = "{}".getBytes(StandardCharsets.UTF_8);
+    IOException checked = new IOException("checked");
+    IOException thrownChecked =
+        expectThrows(
+            IOException.class,
+            () ->
+                json.readUtf8(
+                    bytes,
+                    0,
+                    bytes.length,
+                    reader -> {
+                      throw checked;
+                    }));
+    assertSame(thrownChecked, checked);
+    IllegalStateException runtime = new IllegalStateException("runtime");
+    IllegalStateException thrownRuntime =
+        expectThrows(
+            IllegalStateException.class,
+            () ->
+                json.writeUtf8(
+                    writer -> {
+                      writer.writeObjectStart();
+                      throw runtime;
+                    }));
+    assertSame(thrownRuntime, runtime);
+    assertThrows(
+        ForyJsonException.class,
+        () -> json.readUtf8(bytes, 0, bytes.length, reader -> reader.readInt()));
+
+    // The pooled state was released after every failure and the writer starts clean.
+    assertEquals(
+        new String(json.writeUtf8(writer -> writer.writeInt(1)), StandardCharsets.UTF_8), "1");
+    assertEquals(json.toJson(new Pair()), "{\"id\":0}");
+    assertTrue(
+        json.readUtf8(
+            bytes,
+            0,
+            bytes.length,
+            reader -> reader.consumeNextToken('{') && reader.consumeNextToken('}')));
+  }
+
+  @Test
+  public void readerIsBorrowedFromThePool() {
+    byte[] bytes = "true".getBytes(StandardCharsets.UTF_8);
+    Utf8JsonReader first =
+        json.readUtf8(
+            bytes,
+            0,
+            bytes.length,
+            reader -> {
+              assertTrue(reader.readBoolean());
+              return reader;
+            });
+    Utf8JsonReader second =
+        json.readUtf8(
+            bytes,
+            0,
+            bytes.length,
+            reader -> {
+              assertTrue(reader.readBoolean());
+              return reader;
+            });
+    assertSame(second, first);
+    assertFalse(
+        json.readUtf8("false".getBytes(StandardCharsets.UTF_8), 0, 5, Utf8JsonReader::readBoolean));
+  }
+
+  private static Pair readPair(Utf8JsonReader reader) {
+    Pair pair = new Pair();
+    reader.expectNextToken('{');
+    if (reader.consumeNextToken('}')) {
+      return pair;
+    }
+    do {
+      String name = reader.readFieldName();
+      reader.expect(':');
+      if (name.equals("id")) {
+        pair.id = reader.readInt();
+      } else if (name.equals("name")) {
+        pair.name = reader.readString();
+      } else {
+        reader.skipValue();
+      }
+    } while (reader.consumeCommaOrEndObject());
+    return pair;
+  }
+
+  public static class Pair {
+    public int id;
+    public String name;
+  }
+}