Skip to content

Commit

Permalink
[Improve] spark buildrequest bug fixed.
Browse files Browse the repository at this point in the history
  • Loading branch information
wolfboys committed Sep 29, 2024
1 parent 4f0b394 commit 3c1a49d
Showing 1 changed file with 1 addition and 34 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

package org.apache.streampark.flink.packer.pipeline

import org.apache.streampark.common.conf.{FlinkVersion, SparkVersion, Workspace}
import org.apache.streampark.common.conf.{FlinkVersion, Workspace}
import org.apache.streampark.common.enums.{FlinkDeployMode, FlinkJobType, SparkDeployMode, SparkJobType}
import org.apache.streampark.flink.kubernetes.model.K8sPodTemplates
import org.apache.streampark.flink.packer.docker.DockerConf
Expand Down Expand Up @@ -64,38 +64,6 @@ sealed trait FlinkBuildParam extends BuildParam {

}

sealed trait SparkBuildParam extends BuildParam {

private[this] val localWorkspace = Workspace.local

def workspace: String

def deployMode: SparkDeployMode

def jobType: SparkJobType

def sparkVersion: SparkVersion

def dependencyInfo: DependencyInfo

def customSparkUserJar: String

lazy val providedLibs: DependencyInfo = {
val providedLibs =
ArrayBuffer(localWorkspace.APP_JARS, customSparkUserJar)
if (jobType == SparkJobType.SPARK_SQL) {
providedLibs += s"${localWorkspace.APP_SHIMS}/spark-${sparkVersion.majorVersion}"
}
dependencyInfo.merge(providedLibs.toSet)
}

def getShadedJarPath(rootWorkspace: String): String = {
val safeAppName: String = appName.replaceAll("\\s+", "_")
s"$rootWorkspace/streampark-sparkjob_$safeAppName.jar"
}

}

sealed trait FlinkK8sBuildParam extends FlinkBuildParam {

def clusterId: String
Expand Down Expand Up @@ -163,4 +131,3 @@ case class SparkYarnBuildRequest(
sparkJobType: SparkJobType,
deployMode: SparkDeployMode,
dependencyInfo: DependencyInfo)
extends SparkBuildParam

0 comments on commit 3c1a49d

Please sign in to comment.