From 3c1a49d1b55ec4b9d3adc0e0199b7b24b93bd896 Mon Sep 17 00:00:00 2001 From: benjobs Date: Sun, 29 Sep 2024 12:56:25 +0800 Subject: [PATCH] [Improve] spark buildrequest bug fixed. --- .../flink/packer/pipeline/BuildRequest.scala | 35 +------------------ 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/BuildRequest.scala b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/BuildRequest.scala index 79f0fb595a..ccd710c19c 100644 --- a/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/BuildRequest.scala +++ b/streampark-flink/streampark-flink-packer/src/main/scala/org/apache/streampark/flink/packer/pipeline/BuildRequest.scala @@ -17,7 +17,7 @@ package org.apache.streampark.flink.packer.pipeline -import org.apache.streampark.common.conf.{FlinkVersion, SparkVersion, Workspace} +import org.apache.streampark.common.conf.{FlinkVersion, Workspace} import org.apache.streampark.common.enums.{FlinkDeployMode, FlinkJobType, SparkDeployMode, SparkJobType} import org.apache.streampark.flink.kubernetes.model.K8sPodTemplates import org.apache.streampark.flink.packer.docker.DockerConf @@ -64,38 +64,6 @@ sealed trait FlinkBuildParam extends BuildParam { } -sealed trait SparkBuildParam extends BuildParam { - - private[this] val localWorkspace = Workspace.local - - def workspace: String - - def deployMode: SparkDeployMode - - def jobType: SparkJobType - - def sparkVersion: SparkVersion - - def dependencyInfo: DependencyInfo - - def customSparkUserJar: String - - lazy val providedLibs: DependencyInfo = { - val providedLibs = - ArrayBuffer(localWorkspace.APP_JARS, customSparkUserJar) - if (jobType == SparkJobType.SPARK_SQL) { - providedLibs += s"${localWorkspace.APP_SHIMS}/spark-${sparkVersion.majorVersion}" - } - dependencyInfo.merge(providedLibs.toSet) - } - - def getShadedJarPath(rootWorkspace: String): String = { - val safeAppName: String = appName.replaceAll("\\s+", "_") - s"$rootWorkspace/streampark-sparkjob_$safeAppName.jar" - } - -} - sealed trait FlinkK8sBuildParam extends FlinkBuildParam { def clusterId: String @@ -163,4 +131,3 @@ case class SparkYarnBuildRequest( sparkJobType: SparkJobType, deployMode: SparkDeployMode, dependencyInfo: DependencyInfo) - extends SparkBuildParam