Merge pull request #32 from lyft/DATA-10803

Do not localize s3 files
lyft · Apr 14, 2021 · c879a5d · c879a5d
2 parents 8805e1a + a8aef8f
commit c879a5d
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -493,7 +493,12 @@ private[spark] class Client(
           val localPath = getQualifiedLocalPath(localURI, hadoopConf)
           val linkname = targetDir.map(_ + "/").getOrElse("") +
             destName.orElse(Option(localURI.getFragment())).getOrElse(localPath.getName())
-          val destPath = copyFileToRemote(destDir, localPath, replication, symlinkCache)
+            var destPath = localPath
+          if (!localPath.toUri.getScheme.startsWith("s3")) {
+            destPath = copyFileToRemote(destDir, localPath, replication, symlinkCache)
+          } else {
+            logInfo(s"Adding binary from location: $destPath to the distributed cache")
+          }
           val destFs = FileSystem.get(destPath.toUri(), hadoopConf)
           distCacheMgr.addResource(
             destFs, hadoopConf, destPath, localResources, resType, linkname, statCache,