[SPARK-28098] Fix NPE when specifying parquet files instead of paths (#…

…41)
lyft · Aug 20, 2021 · 4375b8a · 4375b8a
1 parent cefa493
commit 4375b8a
Showing 1 changed file with 6 additions and 4 deletions.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala
@@ -99,18 +99,20 @@ class InMemoryFileIndex(
       new mutable.LinkedHashMap[Path, FileStatus]() ++= files.map(f => f.getPath -> f)
     cachedLeafDirToChildrenFiles =
       if (readPartitionWithSubdirectoryEnabled) {
-        files.toArray.groupBy(file => getRootPathsLeafDir(file.getPath.getParent))
+        files.toArray.groupBy(file => getRootPathsLeafDir(file.getPath.getParent, file.getPath))
       } else {
         files.toArray.groupBy(_.getPath.getParent)
       }
     cachedPartitionSpec = null
   }
 
-  private def getRootPathsLeafDir(path: Path): Path = {
-    if (rootPaths.contains(path)) {
+  private def getRootPathsLeafDir(path: Path, child: Path): Path = {
+    if (rootPaths.contains(child)) {
+      path
+    } else if (rootPaths.contains(path)) {
       path
     } else {
-      getRootPathsLeafDir(path.getParent)
+      getRootPathsLeafDir(path.getParent, path)
     }
   }