[ADH-9623] Upgrade Thrift 0.25: spark4

LICENSE-binary

@@ -344,6 +344,9 @@ org.apache.hive:hive-shims-scheduler
 org.apache.hive:hive-storage-api
 org.apache.httpcomponents:httpclient
 org.apache.httpcomponents:httpcore
+org.apache.httpcomponents.client5:httpclient5
+org.apache.httpcomponents.core5:httpcore5
+org.apache.httpcomponents.core5:httpcore5-h2
 org.apache.ivy:ivy
 org.apache.logging.log4j:log4j-1.2-api
 org.apache.logging.log4j:log4j-api
@@ -360,7 +363,6 @@ org.apache.parquet:parquet-encoding
 org.apache.parquet:parquet-format-structures
 org.apache.parquet:parquet-hadoop
 org.apache.parquet:parquet-jackson
-org.apache.thrift:libfb303
 org.apache.thrift:libthrift
 org.apache.ws.xmlschema:xmlschema-core
 org.apache.xbean:xbean-asm9-shaded

NOTICE-binary

@@ -731,6 +731,12 @@ Copyright 1999-2007 The Apache Software Foundation
 Apache HttpClient
 Copyright 1999-2017 The Apache Software Foundation
 
+Apache HttpComponents Core HTTP/1.1
+Copyright 2005-2021 The Apache Software Foundation
+
+Apache HttpComponents Core HTTP/2
+Copyright 2005-2021 The Apache Software Foundation
+
 Apache Commons Codec
 Copyright 2002-2014 The Apache Software Foundation
 

dev/deps/spark-deps-hadoop-3-hive-2.3

@@ -71,24 +71,27 @@ hadoop-client-api/3.4.3.2-4.4.0-1//hadoop-client-api-3.4.3.2-4.4.0-1.jar
 hadoop-client-runtime/3.4.3.2-4.4.0-1//hadoop-client-runtime-3.4.3.2-4.4.0-1.jar
 hadoop-cloud-storage/3.4.3.2-4.4.0-1//hadoop-cloud-storage-3.4.3.2-4.4.0-1.jar
 hadoop-shaded-guava/1.6.0.1-4.3.0-0//hadoop-shaded-guava-1.6.0.1-4.3.0-0.jar
-hive-beeline/2.3.10.3-4.4.0-2//hive-beeline-2.3.10.3-4.4.0-2.jar
-hive-cli/2.3.10.3-4.4.0-2//hive-cli-2.3.10.3-4.4.0-2.jar
-hive-common/2.3.10.3-4.4.0-2//hive-common-2.3.10.3-4.4.0-2.jar
-hive-exec/2.3.10.3-4.4.0-2/core/hive-exec-2.3.10.3-4.4.0-2-core.jar
-hive-jdbc/2.3.10.3-4.4.0-2//hive-jdbc-2.3.10.3-4.4.0-2.jar
-hive-metastore/2.3.10.3-4.4.0-2//hive-metastore-2.3.10.3-4.4.0-2.jar
-hive-serde/2.3.10.3-4.4.0-2//hive-serde-2.3.10.3-4.4.0-2.jar
-hive-service-rpc/4.3.0.2-4.4.0-0//hive-service-rpc-4.3.0.2-4.4.0-0.jar
-hive-shims-0.23/2.3.10.3-4.4.0-2//hive-shims-0.23-2.3.10.3-4.4.0-2.jar
-hive-shims-common/2.3.10.3-4.4.0-2//hive-shims-common-2.3.10.3-4.4.0-2.jar
-hive-shims-scheduler/2.3.10.3-4.4.0-2//hive-shims-scheduler-2.3.10.3-4.4.0-2.jar
-hive-shims/2.3.10.3-4.4.0-2//hive-shims-2.3.10.3-4.4.0-2.jar
+hive-beeline/2.3.10.4-4.5.0-0//hive-beeline-2.3.10.4-4.5.0-0.jar
+hive-cli/2.3.10.4-4.5.0-0//hive-cli-2.3.10.4-4.5.0-0.jar
+hive-common/2.3.10.4-4.5.0-0//hive-common-2.3.10.4-4.5.0-0.jar
+hive-exec/2.3.10.4-4.5.0-0/core/hive-exec-2.3.10.4-4.5.0-0-core.jar
+hive-jdbc/2.3.10.4-4.5.0-0//hive-jdbc-2.3.10.4-4.5.0-0.jar
+hive-metastore/2.3.10.4-4.5.0-0//hive-metastore-2.3.10.4-4.5.0-0.jar
+hive-serde/2.3.10.4-4.5.0-0//hive-serde-2.3.10.4-4.5.0-0.jar
+hive-service-rpc/4.3.0.3-4.5.0-0//hive-service-rpc-4.3.0.3-4.5.0-0.jar
+hive-shims-0.23/2.3.10.4-4.5.0-0//hive-shims-0.23-2.3.10.4-4.5.0-0.jar
+hive-shims-common/2.3.10.4-4.5.0-0//hive-shims-common-2.3.10.4-4.5.0-0.jar
+hive-shims-scheduler/2.3.10.4-4.5.0-0//hive-shims-scheduler-2.3.10.4-4.5.0-0.jar
+hive-shims/2.3.10.4-4.5.0-0//hive-shims-2.3.10.4-4.5.0-0.jar
 hive-storage-api/2.8.1//hive-storage-api-2.8.1.jar
 hk2-api/3.0.6//hk2-api-3.0.6.jar
 hk2-locator/3.0.6//hk2-locator-3.0.6.jar
 hk2-utils/3.0.6//hk2-utils-3.0.6.jar
 httpclient/4.5.14//httpclient-4.5.14.jar
+httpclient5/5.5//httpclient5-5.5.jar
 httpcore/4.4.16//httpcore-4.4.16.jar
+httpcore5-h2/5.3.4//httpcore5-h2-5.3.4.jar
+httpcore5/5.3.4//httpcore5-5.3.4.jar
 icu4j/78.3//icu4j-78.3.jar
 istack-commons-runtime/4.1.2//istack-commons-runtime-4.1.2.jar
 ivy/2.5.3//ivy-2.5.3.jar
@@ -169,8 +172,7 @@ kubernetes-model-scheduling/7.6.1//kubernetes-model-scheduling-7.6.1.jar
 kubernetes-model-storageclass/7.6.1//kubernetes-model-storageclass-7.6.1.jar
 lapack/3.2.0//lapack-3.2.0.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
-libfb303/0.9.3//libfb303-0.9.3.jar
-libthrift/0.16.0//libthrift-0.16.0.jar
+libthrift/0.25.0//libthrift-0.25.0.jar
 log4j-1.2-api/2.25.4//log4j-1.2-api-2.25.4.jar
 log4j-api/2.25.4//log4j-api-2.25.4.jar
 log4j-core/2.25.4//log4j-core-2.25.4.jar

pom.xml

@@ -138,7 +138,7 @@
     <hive.group>org.apache.hive</hive.group>
     <hive.classifier>core</hive.classifier>
     <!-- Version used in Maven Hive dependency -->
-    <hive.version>2.3.10.3-4.4.0-2</hive.version>
+    <hive.version>2.3.10.4-4.5.0-0</hive.version>
     <!-- note that this should be compatible with Kafka brokers version 0.10 and up -->
     <kafka.version>3.9.2</kafka.version>
     <!-- After 10.17.1.0, the minimum required version is JDK19 -->
@@ -158,7 +158,7 @@
     <orc.classifier>shaded-protobuf</orc.classifier>
     <jetty.version>12.1.10</jetty.version>
     <jakartaservlet.version>6.0.0</jakartaservlet.version>
-    <!-- SPARK-46938: Required by Hive / LibThrift libs -->
+    <!-- SPARK-46938: Required by Hive libs -->
     <javaxservlet.version>4.0.1</javaxservlet.version>
     <chill.version>0.10.0</chill.version>
     <kryo.version>4.0.3</kryo.version>
@@ -194,6 +194,9 @@
     <!--  org.apache.httpcomponents/httpclient-->
     <commons.httpclient.version>4.5.14</commons.httpclient.version>
     <commons.httpcore.version>4.4.16</commons.httpcore.version>
+    <!--  org.apache.httpcomponents.client5/httpclient5, used by libthrift and hive-jdbc -->
+    <httpclient5.version>5.5</httpclient5.version>
+    <httpcore5.version>5.3.4</httpcore5.version>
     <commons.math3.version>3.6.1</commons.math3.version>
     <commons.collections4.version>4.5.0</commons.collections4.version>
     <scala.version>2.13.18</scala.version>
@@ -229,7 +232,7 @@
     <joda.version>2.14.1</joda.version>
     <jsr305.version>3.0.0</jsr305.version>
     <jaxb.version>2.2.11</jaxb.version>
-    <libthrift.version>0.16.0</libthrift.version>
+    <libthrift.version>0.25.0</libthrift.version>
     <antlr4.version>4.13.1</antlr4.version>
     <jpam.version>1.1</jpam.version>
     <selenium.version>4.32.0</selenium.version>
@@ -838,6 +841,21 @@
         <artifactId>httpcore</artifactId>
         <version>${commons.httpcore.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.httpcomponents.client5</groupId>
+        <artifactId>httpclient5</artifactId>
+        <version>${httpclient5.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.httpcomponents.core5</groupId>
+        <artifactId>httpcore5</artifactId>
+        <version>${httpcore5.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.httpcomponents.core5</groupId>
+        <artifactId>httpcore5-h2</artifactId>
+        <version>${httpcore5.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.rocksdb</groupId>
         <artifactId>rocksdbjni</artifactId>
@@ -2263,7 +2281,7 @@
       <dependency>
         <groupId>${hive.group}</groupId>
         <artifactId>hive-service-rpc</artifactId>
-        <version>4.3.0.2-4.4.0-0</version>
+        <version>4.3.0.3-4.5.0-0</version>
         <exclusions>
           <exclusion>
             <groupId>*</groupId>
@@ -2567,20 +2585,10 @@
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
           </exclusion>
+          <!-- libthrift classes do not use jakarta.annotation -->
           <exclusion>
-            <groupId>javax.annotation</groupId>
-            <artifactId>javax.annotation-api</artifactId>
-          </exclusion>
-        </exclusions>
-      </dependency>
-      <dependency>
-        <groupId>org.apache.thrift</groupId>
-        <artifactId>libfb303</artifactId>
-        <version>0.9.3</version>
-        <exclusions>
-          <exclusion>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
+            <groupId>jakarta.annotation</groupId>
+            <artifactId>jakarta.annotation-api</artifactId>
           </exclusion>
         </exclusions>
       </dependency>

sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java

@@ -35,12 +35,15 @@
 import org.apache.hive.service.rpc.thrift.TGetQueryIdReq;
 import org.apache.hive.service.rpc.thrift.TGetQueryIdResp;
 import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup;
+import org.apache.thrift.TConfiguration;
 import org.apache.thrift.TException;
 import org.apache.thrift.TProcessorFactory;
 import org.apache.thrift.protocol.TBinaryProtocol;
 import org.apache.thrift.server.TServer;
 import org.apache.thrift.server.TThreadPoolServer;
 import org.apache.thrift.transport.TServerSocket;
+import org.apache.thrift.transport.TSocket;
+import org.apache.thrift.transport.TTransportException;
 import org.apache.thrift.transport.TTransportFactory;
 
 import org.apache.spark.sql.hive.thriftserver.HiveThriftServer2$;
@@ -91,7 +94,18 @@ protected void initializeServer() {
 
       // Server args
       int maxMessageSize = hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE);
-      TThreadPoolServer.Args sargs = new TThreadPoolServer.Args(serverSocket)
+      TThreadPoolServer.Args sargs = new TThreadPoolServer.Args(
+          new TServerSocket(serverSocket.getServerSocket()) {
+            @Override
+            public TSocket accept() throws TTransportException {
+              TSocket socket = super.accept();
+              // A SASL frame carries a whole message, and Thrift rejects frames above the max frame size.
+              TConfiguration configuration = socket.getConfiguration();
+              configuration.setMaxFrameSize(
+                  Math.max(configuration.getMaxFrameSize(), configuration.getMaxMessageSize()));
+              return socket;
+            }
+          })
           .processorFactory(processorFactory).transportFactory(transportFactory)
           .protocolFactory(new TBinaryProtocol.Factory())
           .inputProtocolFactory(new TBinaryProtocol.Factory(true, true, maxMessageSize, maxMessageSize))

sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala

@@ -28,11 +28,11 @@ import scala.util.Try
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
 import org.apache.hadoop.hive.ql.metadata.Hive
 import org.apache.hadoop.hive.ql.session.SessionState
+import org.apache.hc.client5.http.impl.classic.HttpClientBuilder
 import org.apache.hive.jdbc.HttpBasicAuthInterceptor
 import org.apache.hive.service.auth.PlainSaslHelper
 import org.apache.hive.service.cli.thrift.{ThriftBinaryCLIService, ThriftCLIService, ThriftCLIServiceClient}
 import org.apache.hive.service.rpc.thrift.TCLIService.Client
-import org.apache.http.impl.client.HttpClientBuilder
 import org.apache.thrift.protocol.TBinaryProtocol
 import org.apache.thrift.transport.{THttpClient, TSocket}
 
@@ -116,7 +116,7 @@ trait SharedThriftServer extends SharedSparkSession {
           null, null, true, new util.HashMap[String, String]())
         new THttpClient(
           s"http://localhost:$serverPort/cliservice",
-          HttpClientBuilder.create.addInterceptorFirst(interceptor).build())
+          HttpClientBuilder.create.addRequestInterceptorFirst(interceptor).build())
     }
 
     val protocol = new TBinaryProtocol(transport)

sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala

@@ -21,6 +21,7 @@ import java.sql.{ResultSet, SQLException}
 import java.util.concurrent.atomic.AtomicBoolean
 
 import org.apache.hive.service.cli.{GetInfoType, HiveSQLException, OperationHandle}
+import org.apache.thrift.TConfiguration
 
 import org.apache.spark.{ErrorMessageFormat, TaskKilled}
 import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd}
@@ -270,6 +271,18 @@ trait ThriftServerWithSparkContextSuite extends SharedThriftServer {
       }
     }
   }
+
+  test("Messages larger than Thrift's default max frame size") {
+    // A SASL frame carries a whole message; the statement and the result below exceed the default
+    // max frame size.
+    val length = TConfiguration.DEFAULT_MAX_FRAME_SIZE + 1024
+    withJdbcStatement() { stmt =>
+      stmt.execute(s"SET spark.test.thrift.large.value=${"x" * length}")
+      val rs = stmt.executeQuery(s"SELECT repeat('x', $length)")
+      assert(rs.next())
+      assert(rs.getString(1).length === length)
+    }
+  }
 }
 
 class ThriftServerWithSparkContextInBinarySuite extends ThriftServerWithSparkContextSuite {

sql/hive/pom.xml

@@ -181,10 +181,6 @@
       <groupId>org.apache.thrift</groupId>
       <artifactId>libthrift</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.apache.thrift</groupId>
-      <artifactId>libfb303</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.apache.derby</groupId>
       <artifactId>derby</artifactId>

sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala

@@ -1506,6 +1506,9 @@ private[hive] object HiveClientImpl extends Logging {
               // apply the updated maxMessageSize
               tTransport.updateKnownMessageSize(0L)
             }
+            // A SASL frame carries a whole message, and Thrift rejects frames above the max frame
+            // size.
+            tConf.setMaxFrameSize(math.max(tConf.getMaxFrameSize, tConf.getMaxMessageSize))
           case _ =>
         }
         configure(msc.getTTransport)

sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala

@@ -74,7 +74,7 @@ package object client {
 
     case object v2_3_arenadata extends HiveVersion("2.3.10",
       exclusions = v2_3.exclusions :+ "org.apache.hive:hive-llap-tez",
-      mavenVersionOverride = Some("2.3.10.3-4.4.0-2"))
+      mavenVersionOverride = Some("2.3.10.4-4.5.0-0"))
 
     // Since Hive 3.0, HookUtils uses org.apache.logging.log4j.util.Strings
     // Since HIVE-14496, Hive.java uses calcite-core