diff --git a/LICENSE-binary b/LICENSE-binary index 958903f013965..4fd568b227d49 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -344,6 +344,9 @@ org.apache.hive:hive-shims-scheduler org.apache.hive:hive-storage-api org.apache.httpcomponents:httpclient org.apache.httpcomponents:httpcore +org.apache.httpcomponents.client5:httpclient5 +org.apache.httpcomponents.core5:httpcore5 +org.apache.httpcomponents.core5:httpcore5-h2 org.apache.ivy:ivy org.apache.logging.log4j:log4j-1.2-api org.apache.logging.log4j:log4j-api @@ -360,7 +363,6 @@ org.apache.parquet:parquet-encoding org.apache.parquet:parquet-format-structures org.apache.parquet:parquet-hadoop org.apache.parquet:parquet-jackson -org.apache.thrift:libfb303 org.apache.thrift:libthrift org.apache.ws.xmlschema:xmlschema-core org.apache.xbean:xbean-asm9-shaded diff --git a/NOTICE-binary b/NOTICE-binary index 69ade56b36c45..864fe5927210c 100644 --- a/NOTICE-binary +++ b/NOTICE-binary @@ -731,6 +731,12 @@ Copyright 1999-2007 The Apache Software Foundation Apache HttpClient Copyright 1999-2017 The Apache Software Foundation +Apache HttpComponents Core HTTP/1.1 +Copyright 2005-2021 The Apache Software Foundation + +Apache HttpComponents Core HTTP/2 +Copyright 2005-2021 The Apache Software Foundation + Apache Commons Codec Copyright 2002-2014 The Apache Software Foundation diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index a014148ee34c1..f5ba0023db199 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -71,24 +71,27 @@ hadoop-client-api/3.4.3.2-4.4.0-1//hadoop-client-api-3.4.3.2-4.4.0-1.jar hadoop-client-runtime/3.4.3.2-4.4.0-1//hadoop-client-runtime-3.4.3.2-4.4.0-1.jar hadoop-cloud-storage/3.4.3.2-4.4.0-1//hadoop-cloud-storage-3.4.3.2-4.4.0-1.jar hadoop-shaded-guava/1.6.0.1-4.3.0-0//hadoop-shaded-guava-1.6.0.1-4.3.0-0.jar -hive-beeline/2.3.10.3-4.4.0-2//hive-beeline-2.3.10.3-4.4.0-2.jar -hive-cli/2.3.10.3-4.4.0-2//hive-cli-2.3.10.3-4.4.0-2.jar -hive-common/2.3.10.3-4.4.0-2//hive-common-2.3.10.3-4.4.0-2.jar -hive-exec/2.3.10.3-4.4.0-2/core/hive-exec-2.3.10.3-4.4.0-2-core.jar -hive-jdbc/2.3.10.3-4.4.0-2//hive-jdbc-2.3.10.3-4.4.0-2.jar -hive-metastore/2.3.10.3-4.4.0-2//hive-metastore-2.3.10.3-4.4.0-2.jar -hive-serde/2.3.10.3-4.4.0-2//hive-serde-2.3.10.3-4.4.0-2.jar -hive-service-rpc/4.3.0.2-4.4.0-0//hive-service-rpc-4.3.0.2-4.4.0-0.jar -hive-shims-0.23/2.3.10.3-4.4.0-2//hive-shims-0.23-2.3.10.3-4.4.0-2.jar -hive-shims-common/2.3.10.3-4.4.0-2//hive-shims-common-2.3.10.3-4.4.0-2.jar -hive-shims-scheduler/2.3.10.3-4.4.0-2//hive-shims-scheduler-2.3.10.3-4.4.0-2.jar -hive-shims/2.3.10.3-4.4.0-2//hive-shims-2.3.10.3-4.4.0-2.jar +hive-beeline/2.3.10.4-4.5.0-0//hive-beeline-2.3.10.4-4.5.0-0.jar +hive-cli/2.3.10.4-4.5.0-0//hive-cli-2.3.10.4-4.5.0-0.jar +hive-common/2.3.10.4-4.5.0-0//hive-common-2.3.10.4-4.5.0-0.jar +hive-exec/2.3.10.4-4.5.0-0/core/hive-exec-2.3.10.4-4.5.0-0-core.jar +hive-jdbc/2.3.10.4-4.5.0-0//hive-jdbc-2.3.10.4-4.5.0-0.jar +hive-metastore/2.3.10.4-4.5.0-0//hive-metastore-2.3.10.4-4.5.0-0.jar +hive-serde/2.3.10.4-4.5.0-0//hive-serde-2.3.10.4-4.5.0-0.jar +hive-service-rpc/4.3.0.3-4.5.0-0//hive-service-rpc-4.3.0.3-4.5.0-0.jar +hive-shims-0.23/2.3.10.4-4.5.0-0//hive-shims-0.23-2.3.10.4-4.5.0-0.jar +hive-shims-common/2.3.10.4-4.5.0-0//hive-shims-common-2.3.10.4-4.5.0-0.jar +hive-shims-scheduler/2.3.10.4-4.5.0-0//hive-shims-scheduler-2.3.10.4-4.5.0-0.jar +hive-shims/2.3.10.4-4.5.0-0//hive-shims-2.3.10.4-4.5.0-0.jar hive-storage-api/2.8.1//hive-storage-api-2.8.1.jar hk2-api/3.0.6//hk2-api-3.0.6.jar hk2-locator/3.0.6//hk2-locator-3.0.6.jar hk2-utils/3.0.6//hk2-utils-3.0.6.jar httpclient/4.5.14//httpclient-4.5.14.jar +httpclient5/5.5//httpclient5-5.5.jar httpcore/4.4.16//httpcore-4.4.16.jar +httpcore5-h2/5.3.4//httpcore5-h2-5.3.4.jar +httpcore5/5.3.4//httpcore5-5.3.4.jar icu4j/78.3//icu4j-78.3.jar istack-commons-runtime/4.1.2//istack-commons-runtime-4.1.2.jar ivy/2.5.3//ivy-2.5.3.jar @@ -169,8 +172,7 @@ kubernetes-model-scheduling/7.6.1//kubernetes-model-scheduling-7.6.1.jar kubernetes-model-storageclass/7.6.1//kubernetes-model-storageclass-7.6.1.jar lapack/3.2.0//lapack-3.2.0.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar -libfb303/0.9.3//libfb303-0.9.3.jar -libthrift/0.16.0//libthrift-0.16.0.jar +libthrift/0.25.0//libthrift-0.25.0.jar log4j-1.2-api/2.25.4//log4j-1.2-api-2.25.4.jar log4j-api/2.25.4//log4j-api-2.25.4.jar log4j-core/2.25.4//log4j-core-2.25.4.jar diff --git a/pom.xml b/pom.xml index b9cabd98dfa41..94806f012edb1 100644 --- a/pom.xml +++ b/pom.xml @@ -138,7 +138,7 @@ org.apache.hive core - 2.3.10.3-4.4.0-2 + 2.3.10.4-4.5.0-0 3.9.2 @@ -158,7 +158,7 @@ shaded-protobuf 12.1.10 6.0.0 - + 4.0.1 0.10.0 4.0.3 @@ -194,6 +194,9 @@ 4.5.14 4.4.16 + + 5.5 + 5.3.4 3.6.1 4.5.0 2.13.18 @@ -229,7 +232,7 @@ 2.14.1 3.0.0 2.2.11 - 0.16.0 + 0.25.0 4.13.1 1.1 4.32.0 @@ -838,6 +841,21 @@ httpcore ${commons.httpcore.version} + + org.apache.httpcomponents.client5 + httpclient5 + ${httpclient5.version} + + + org.apache.httpcomponents.core5 + httpcore5 + ${httpcore5.version} + + + org.apache.httpcomponents.core5 + httpcore5-h2 + ${httpcore5.version} + org.rocksdb rocksdbjni @@ -2263,7 +2281,7 @@ ${hive.group} hive-service-rpc - 4.3.0.2-4.4.0-0 + 4.3.0.3-4.5.0-0 * @@ -2567,20 +2585,10 @@ org.slf4j slf4j-api + - javax.annotation - javax.annotation-api - - - - - org.apache.thrift - libfb303 - 0.9.3 - - - org.slf4j - slf4j-api + jakarta.annotation + jakarta.annotation-api diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java index c7fa7b5f3e0ac..5f18612ca15c7 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java @@ -35,12 +35,15 @@ import org.apache.hive.service.rpc.thrift.TGetQueryIdReq; import org.apache.hive.service.rpc.thrift.TGetQueryIdResp; import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup; +import org.apache.thrift.TConfiguration; import org.apache.thrift.TException; import org.apache.thrift.TProcessorFactory; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.server.TServer; import org.apache.thrift.server.TThreadPoolServer; import org.apache.thrift.transport.TServerSocket; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransportException; import org.apache.thrift.transport.TTransportFactory; import org.apache.spark.sql.hive.thriftserver.HiveThriftServer2$; @@ -91,7 +94,18 @@ protected void initializeServer() { // Server args int maxMessageSize = hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE); - TThreadPoolServer.Args sargs = new TThreadPoolServer.Args(serverSocket) + TThreadPoolServer.Args sargs = new TThreadPoolServer.Args( + new TServerSocket(serverSocket.getServerSocket()) { + @Override + public TSocket accept() throws TTransportException { + TSocket socket = super.accept(); + // A SASL frame carries a whole message, and Thrift rejects frames above the max frame size. + TConfiguration configuration = socket.getConfiguration(); + configuration.setMaxFrameSize( + Math.max(configuration.getMaxFrameSize(), configuration.getMaxMessageSize())); + return socket; + } + }) .processorFactory(processorFactory).transportFactory(transportFactory) .protocolFactory(new TBinaryProtocol.Factory()) .inputProtocolFactory(new TBinaryProtocol.Factory(true, true, maxMessageSize, maxMessageSize)) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala index 6f0fedcb85368..86209fe791e37 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala @@ -28,11 +28,11 @@ import scala.util.Try import org.apache.hadoop.hive.conf.HiveConf.ConfVars import org.apache.hadoop.hive.ql.metadata.Hive import org.apache.hadoop.hive.ql.session.SessionState +import org.apache.hc.client5.http.impl.classic.HttpClientBuilder import org.apache.hive.jdbc.HttpBasicAuthInterceptor import org.apache.hive.service.auth.PlainSaslHelper import org.apache.hive.service.cli.thrift.{ThriftBinaryCLIService, ThriftCLIService, ThriftCLIServiceClient} import org.apache.hive.service.rpc.thrift.TCLIService.Client -import org.apache.http.impl.client.HttpClientBuilder import org.apache.thrift.protocol.TBinaryProtocol import org.apache.thrift.transport.{THttpClient, TSocket} @@ -116,7 +116,7 @@ trait SharedThriftServer extends SharedSparkSession { null, null, true, new util.HashMap[String, String]()) new THttpClient( s"http://localhost:$serverPort/cliservice", - HttpClientBuilder.create.addInterceptorFirst(interceptor).build()) + HttpClientBuilder.create.addRequestInterceptorFirst(interceptor).build()) } val protocol = new TBinaryProtocol(transport) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala index 2374865b1a0da..344f55fad7738 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationSuite.scala @@ -601,8 +601,8 @@ class SparkMetadataOperationSuite extends HiveThriftServer2TestBase { assert(metaData.getDriverVersion === HiveVersionInfo.getVersion) assert(metaData.getDatabaseMajorVersion === VersionUtils.majorVersion(SPARK_VERSION)) assert(metaData.getDatabaseMinorVersion === VersionUtils.minorVersion(SPARK_VERSION)) - assert(metaData.getIdentifierQuoteString === " ", - "This method returns a space \" \" if identifier quoting is not supported") + assert(metaData.getIdentifierQuoteString === "`", + "The Arenadata Hive JDBC driver quotes identifiers with backticks") assert(metaData.getNumericFunctions === "") assert(metaData.getStringFunctions === "") assert(metaData.getSystemFunctions === "") @@ -656,7 +656,7 @@ class SparkMetadataOperationSuite extends HiveThriftServer2TestBase { assert(metaData.supportsResultSetType(ResultSet.TYPE_FORWARD_ONLY)) assert(metaData.supportsResultSetType(ResultSet.TYPE_SCROLL_INSENSITIVE)) assert(metaData.supportsResultSetType(ResultSet.TYPE_SCROLL_SENSITIVE)) - assert(!metaData.supportsBatchUpdates) + assert(metaData.supportsBatchUpdates) assert(!metaData.getUDTs(",", "%", "%", null).next()) assert(!metaData.supportsSavepoints) assert(!metaData.supportsResultSetHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT)) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala index 1ecf5b3dae4a0..a40f43035fbf0 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala @@ -21,6 +21,7 @@ import java.sql.{ResultSet, SQLException} import java.util.concurrent.atomic.AtomicBoolean import org.apache.hive.service.cli.{GetInfoType, HiveSQLException, OperationHandle} +import org.apache.thrift.TConfiguration import org.apache.spark.{ErrorMessageFormat, TaskKilled} import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd} @@ -270,6 +271,18 @@ trait ThriftServerWithSparkContextSuite extends SharedThriftServer { } } } + + test("Messages larger than Thrift's default max frame size") { + // A SASL frame carries a whole message; the statement and the result below exceed the default + // max frame size. + val length = TConfiguration.DEFAULT_MAX_FRAME_SIZE + 1024 + withJdbcStatement() { stmt => + stmt.execute(s"SET spark.test.thrift.large.value=${"x" * length}") + val rs = stmt.executeQuery(s"SELECT repeat('x', $length)") + assert(rs.next()) + assert(rs.getString(1).length === length) + } + } } class ThriftServerWithSparkContextInBinarySuite extends ThriftServerWithSparkContextSuite { diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 3725ade61c9af..bf12a3ef1f122 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -181,10 +181,6 @@ org.apache.thrift libthrift - - org.apache.thrift - libfb303 - org.apache.derby derby diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 4bebb6c85704a..3d54d9b785988 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -1506,6 +1506,9 @@ private[hive] object HiveClientImpl extends Logging { // apply the updated maxMessageSize tTransport.updateKnownMessageSize(0L) } + // A SASL frame carries a whole message, and Thrift rejects frames above the max frame + // size. + tConf.setMaxFrameSize(math.max(tConf.getMaxFrameSize, tConf.getMaxMessageSize)) case _ => } configure(msc.getTTransport) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala index f31a7ea992858..bc089dc596878 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala @@ -74,7 +74,7 @@ package object client { case object v2_3_arenadata extends HiveVersion("2.3.10", exclusions = v2_3.exclusions :+ "org.apache.hive:hive-llap-tez", - mavenVersionOverride = Some("2.3.10.3-4.4.0-2")) + mavenVersionOverride = Some("2.3.10.4-4.5.0-0")) // Since Hive 3.0, HookUtils uses org.apache.logging.log4j.util.Strings // Since HIVE-14496, Hive.java uses calcite-core