Merge pull request #19 from acceldata-io/ODP-2032

ODP-2032 Spark3 CVE's reconciliation from ODP-3.2.3.2-3 to ODP-main
acceldata-io · Aug 20, 2024 · 136da58 · 136da58
2 parents dc523a5 + 2aa8743
commit 136da58
Show file tree

Hide file tree

Showing 10 changed files with 50 additions and 96 deletions.
diff --git a/core/pom.xml b/core/pom.xml
@@ -479,14 +479,6 @@
           <groupId>commons-logging</groupId>
           <artifactId>commons-logging</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-mapper-asl</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-core-asl</artifactId>
-        </exclusion>
         <exclusion>
           <groupId>com.fasterxml.jackson.core</groupId>
           <artifactId>jackson-core</artifactId>

diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -97,16 +97,14 @@ httpclient/4.5.13//httpclient-4.5.13.jar
 httpcore/4.4.14//httpcore-4.4.14.jar
 ini4j/0.5.4//ini4j-0.5.4.jar
 istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar
-ivy/2.5.0//ivy-2.5.0.jar
-jackson-annotations/2.13.4//jackson-annotations-2.13.4.jar
-jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
-jackson-core/2.13.4//jackson-core-2.13.4.jar
-jackson-databind/2.13.4.1//jackson-databind-2.13.4.1.jar
-jackson-dataformat-cbor/2.13.4//jackson-dataformat-cbor-2.13.4.jar
-jackson-dataformat-yaml/2.13.4//jackson-dataformat-yaml-2.13.4.jar
-jackson-datatype-jsr310/2.13.4//jackson-datatype-jsr310-2.13.4.jar
-jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
-jackson-module-scala_2.12/2.13.4//jackson-module-scala_2.12-2.13.4.jar
+ivy/2.5.1//ivy-2.5.1.jar
+jackson-annotations/2.14.2//jackson-annotations-2.14.2.jar
+jackson-core/2.14.2//jackson-core-2.14.2.jar
+jackson-databind/2.14.2//jackson-databind-2.14.2.jar
+jackson-dataformat-cbor/2.14.2//jackson-dataformat-cbor-2.14.2.jar
+jackson-dataformat-yaml/2.14.2//jackson-dataformat-yaml-2.14.2.jar
+jackson-datatype-jsr310/2.14.2//jackson-datatype-jsr310-2.14.2.jar
+jackson-module-scala_2.12/2.14.2//jackson-module-scala_2.12-2.14.2.jar
 jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar
 jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar
 jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar
@@ -144,31 +142,32 @@ jsr305/3.0.0//jsr305-3.0.0.jar
 jta/1.1//jta-1.1.jar
 jul-to-slf4j/2.0.3//jul-to-slf4j-2.0.3.jar
 kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client-api/6.1.1//kubernetes-client-api-6.1.1.jar
-kubernetes-client/6.1.1//kubernetes-client-6.1.1.jar
-kubernetes-httpclient-okhttp/6.1.1//kubernetes-httpclient-okhttp-6.1.1.jar
-kubernetes-model-admissionregistration/6.1.1//kubernetes-model-admissionregistration-6.1.1.jar
-kubernetes-model-apiextensions/6.1.1//kubernetes-model-apiextensions-6.1.1.jar
-kubernetes-model-apps/6.1.1//kubernetes-model-apps-6.1.1.jar
-kubernetes-model-autoscaling/6.1.1//kubernetes-model-autoscaling-6.1.1.jar
-kubernetes-model-batch/6.1.1//kubernetes-model-batch-6.1.1.jar
-kubernetes-model-certificates/6.1.1//kubernetes-model-certificates-6.1.1.jar
-kubernetes-model-common/6.1.1//kubernetes-model-common-6.1.1.jar
-kubernetes-model-coordination/6.1.1//kubernetes-model-coordination-6.1.1.jar
-kubernetes-model-core/6.1.1//kubernetes-model-core-6.1.1.jar
-kubernetes-model-discovery/6.1.1//kubernetes-model-discovery-6.1.1.jar
-kubernetes-model-events/6.1.1//kubernetes-model-events-6.1.1.jar
-kubernetes-model-extensions/6.1.1//kubernetes-model-extensions-6.1.1.jar
-kubernetes-model-flowcontrol/6.1.1//kubernetes-model-flowcontrol-6.1.1.jar
-kubernetes-model-gatewayapi/6.1.1//kubernetes-model-gatewayapi-6.1.1.jar
-kubernetes-model-metrics/6.1.1//kubernetes-model-metrics-6.1.1.jar
-kubernetes-model-networking/6.1.1//kubernetes-model-networking-6.1.1.jar
-kubernetes-model-node/6.1.1//kubernetes-model-node-6.1.1.jar
-kubernetes-model-policy/6.1.1//kubernetes-model-policy-6.1.1.jar
-kubernetes-model-rbac/6.1.1//kubernetes-model-rbac-6.1.1.jar
-kubernetes-model-scheduling/6.1.1//kubernetes-model-scheduling-6.1.1.jar
-kubernetes-model-storageclass/6.1.1//kubernetes-model-storageclass-6.1.1.jar
-lapack/3.0.2//lapack-3.0.2.jar
+kubernetes-client-api/6.8.1//kubernetes-client-api-6.8.1.jar
+kubernetes-client/6.8.1//kubernetes-client-6.8.1.jar
+kubernetes-httpclient-okhttp/6.8.1//kubernetes-httpclient-okhttp-6.8.1.jar
+kubernetes-model-admissionregistration/6.8.1//kubernetes-model-admissionregistration-6.8.1.jar
+kubernetes-model-apiextensions/6.8.1//kubernetes-model-apiextensions-6.8.1.jar
+kubernetes-model-apps/6.8.1//kubernetes-model-apps-6.8.1.jar
+kubernetes-model-autoscaling/6.8.1//kubernetes-model-autoscaling-6.8.1.jar
+kubernetes-model-batch/6.8.1//kubernetes-model-batch-6.8.1.jar
+kubernetes-model-certificates/6.8.1//kubernetes-model-certificates-6.8.1.jar
+kubernetes-model-common/6.8.1//kubernetes-model-common-6.8.1.jar
+kubernetes-model-coordination/6.8.1//kubernetes-model-coordination-6.8.1.jar
+kubernetes-model-core/6.8.1//kubernetes-model-core-6.8.1.jar
+kubernetes-model-discovery/6.8.1//kubernetes-model-discovery-6.8.1.jar
+kubernetes-model-events/6.8.1//kubernetes-model-events-6.8.1.jar
+kubernetes-model-extensions/6.8.1//kubernetes-model-extensions-6.8.1.jar
+kubernetes-model-flowcontrol/6.8.1//kubernetes-model-flowcontrol-6.8.1.jar
+kubernetes-model-gatewayapi/6.8.1//kubernetes-model-gatewayapi-6.8.1.jar
+kubernetes-model-metrics/6.8.1//kubernetes-model-metrics-6.8.1.jar
+kubernetes-model-networking/6.8.1//kubernetes-model-networking-6.8.1.jar
+kubernetes-model-node/6.8.1//kubernetes-model-node-6.8.1.jar
+kubernetes-model-policy/6.8.1//kubernetes-model-policy-6.8.1.jar
+kubernetes-model-rbac/6.8.1//kubernetes-model-rbac-6.8.1.jar
+kubernetes-model-resource/6.8.1//kubernetes-model-resource-6.8.1.jar
+kubernetes-model-scheduling/6.8.1//kubernetes-model-scheduling-6.8.1.jar
+kubernetes-model-storageclass/6.8.1//kubernetes-model-storageclass-6.8.1.jar
+lapack/3.0.3//lapack-3.0.3.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 libfb303/0.9.3//libfb303-0.9.3.jar
 libthrift/0.12.0//libthrift-0.12.0.jar

diff --git a/pom.xml b/pom.xml
@@ -1307,10 +1307,6 @@
             <groupId>asm</groupId>
             <artifactId>asm</artifactId>
           </exclusion>
-          <exclusion>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-mapper-asl</artifactId>
-          </exclusion>
           <exclusion>
             <groupId>org.ow2.asm</groupId>
             <artifactId>asm</artifactId>
@@ -1807,28 +1803,7 @@
           </exclusion>
         </exclusions>
       </dependency>
-      <dependency>
-        <groupId>org.codehaus.jackson</groupId>
-        <artifactId>jackson-core-asl</artifactId>
-        <version>${codehaus.jackson.version}</version>
-        <scope>${hadoop.deps.scope}</scope>
-      </dependency>
-      <dependency>
-        <groupId>org.codehaus.jackson</groupId>
-        <artifactId>jackson-mapper-asl</artifactId>
-        <version>${codehaus.jackson.version}</version>
-        <scope>${hadoop.deps.scope}</scope>
-      </dependency>
-      <dependency>
-        <groupId>org.codehaus.jackson</groupId>
-        <artifactId>jackson-xc</artifactId>
-        <version>${codehaus.jackson.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.codehaus.jackson</groupId>
-        <artifactId>jackson-jaxrs</artifactId>
-        <version>${codehaus.jackson.version}</version>
-      </dependency>
+      <!-- Hive 2.3 need this to init Hive's FunctionRegistry -->
       <dependency>
         <groupId>${hive.group}</groupId>
         <artifactId>hive-beeline</artifactId>

diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
@@ -201,6 +201,16 @@
        <artifactId>jersey-json</artifactId>
        <scope>test</scope>
        <version>${jersey-1.version}</version>
+       <exclusions>
+         <exclusion>
+           <groupId>org.codehaus.jackson</groupId>
+           <artifactId>jackson-core-asl</artifactId>
+         </exclusion>
+         <exclusion>
+           <groupId>org.codehaus.jackson</groupId>
+           <artifactId>jackson-mapper-asl</artifactId>
+         </exclusion>
+       </exclusions>
      </dependency>
      <dependency>
        <groupId>com.sun.jersey</groupId>

diff --git a/sql/core/pom.xml b/sql/core/pom.xml
@@ -272,16 +272,6 @@
     <profile>
       <id>hadoop-2</id>
       <dependencies>
-        <dependency>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-core-asl</artifactId>
-          <scope>test</scope>
-        </dependency>
-        <dependency>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-mapper-asl</artifactId>
-          <scope>test</scope>
-        </dependency>
       </dependencies>
     </profile>
   </profiles>

diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/CLIService.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/CLIService.java
@@ -534,7 +534,7 @@ public synchronized String getDelegationTokenFromMetaStore(String owner)
 
     try {
       Hive.closeCurrent();
-      return Hive.get(hiveConf).getDelegationToken(owner, owner);
+      return Hive.getWithoutRegisterFns(hiveConf).getDelegationToken(owner, owner);
     } catch (HiveException e) {
       if (e.getCause() instanceof UnsupportedOperationException) {
         throw (UnsupportedOperationException)e.getCause();

diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
@@ -252,7 +252,7 @@ public static int setVariable(String varname, String varvalue) throws Exception
       ss.getHiveVariables().put(propName, substitution.substitute(ss.getConf(),varvalue));
     } else if (varname.startsWith(METACONF_PREFIX)) {
       String propName = varname.substring(METACONF_PREFIX.length());
-      Hive hive = Hive.get(ss.getConf());
+      Hive hive = Hive.getWithoutRegisterFns(ss.getConf());
       hive.setMetaConf(propName, substitution.substitute(ss.getConf(), varvalue));
     } else {
       setConf(varname, varname, varvalue, true);
@@ -413,7 +413,7 @@ public HiveConf getHiveConf() {
   @Override
   public IMetaStoreClient getMetaStoreClient() throws HiveSQLException {
     try {
-      return Hive.get(getHiveConf()).getMSC();
+      return Hive.getWithoutRegisterFns(getHiveConf()).getMSC();
     } catch (HiveException e) {
       throw new HiveSQLException("Failed to get metastore connection", e);
     } catch (MetaException e) {

diff --git a/...hriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java b/...hriftserver/src/main/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java
@@ -54,7 +54,7 @@ public HiveSessionImplwithUGI(TProtocolVersion protocol, String username, String
     // create a new metastore connection for this particular user session
     Hive.set(null);
     try {
-      sessionHive = Hive.get(getHiveConf());
+      sessionHive = Hive.getWithoutRegisterFns(getHiveConf());
     } catch (HiveException e) {
       throw new HiveSQLException("Failed to setup metastore connection", e);
     }
@@ -140,7 +140,7 @@ private void setDelegationToken(String delegationTokenStr) throws HiveSQLExcepti
   private void cancelDelegationToken() throws HiveSQLException {
     if (delegationTokenStr != null) {
       try {
-        Hive.get(getHiveConf()).cancelDelegationToken(delegationTokenStr);
+        Hive.getWithoutRegisterFns(getHiveConf()).cancelDelegationToken(delegationTokenStr);
       } catch (HiveException e) {
         throw new HiveSQLException("Couldn't cancel delegation token", e);
       }

diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
@@ -143,10 +143,6 @@
       <groupId>org.apache.httpcomponents</groupId>
       <artifactId>httpclient</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.codehaus.jackson</groupId>
-      <artifactId>jackson-mapper-asl</artifactId>
-    </dependency>
     <!-- transitive dependencies of hive-exec-core doesn't declare -->
     <dependency>
       <groupId>commons-codec</groupId>

diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -19,15 +19,13 @@ package org.apache.spark.sql.hive.test
 
 import java.io.File
 import java.net.URI
-import java.util.{Set => JavaSet}
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
-import org.apache.hadoop.hive.ql.exec.FunctionRegistry
 import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe
 
 import org.apache.spark.{SparkConf, SparkContext}
@@ -523,12 +521,6 @@ private[hive] class TestHiveSparkSession(
     }
   }
 
-  /**
-   * Records the UDFs present when the server starts, so we can delete ones that are created by
-   * tests.
-   */
-  protected val originalUDFs: JavaSet[String] = FunctionRegistry.getFunctionNames
-
   /**
    * Resets the test instance by deleting any table, view, temp view, and UDF that have been created
    */