remove fsspec bug workarounds (#39)

capitalone · Mar 10, 2021 · 15744ed · 15744ed
1 parent e898d59
commit 15744ed
Show file tree

Hide file tree

Showing 2 changed files with 0 additions and 45 deletions.
diff --git a/rubicon/repository/memory.py b/rubicon/repository/memory.py
@@ -1,4 +1,3 @@
-import os
 import pickle
 
 import fsspec
@@ -31,38 +30,6 @@ def __init__(self, root_dir=None):
 
         self.filesystem.mkdir(self.root_dir)
 
-    def _add_dirnames_to_pseudo_dirs(self, path):
-        """Adds each persisted in-memory file's directory to
-        `fsspec`'s `pseudo_dirs`.
-
-        This is a workaround to a bug in `fsspec` where the
-        memory filesystem does not properly populate the
-        `pseudo_dirs` list.
-        """
-        pseudo_dirs = set(self.filesystem.pseudo_dirs)
-
-        while path != "" and path != "/":
-            path = os.path.dirname(path)
-            pseudo_dirs.add(path)
-
-        self.filesystem.pseudo_dirs = list(pseudo_dirs)
-
-    def _persist_bytes(self, bytes_data, path):
-        """Persists the Rubicon object `domain` to the
-        in-memory path defined by `path`.
-        """
-        super()._persist_bytes(bytes_data, path)
-
-        self._add_dirnames_to_pseudo_dirs(path)
-
-    def _persist_domain(self, domain, path):
-        """Persists the Rubicon object `domain` to the
-        in-memory path defined by `path`.
-        """
-        super()._persist_domain(domain, path)
-
-        self._add_dirnames_to_pseudo_dirs(path)
-
     def _persist_dataframe(self, df, path):
         """Persists the `dask` dataframe `df` to the in-memory
         path defined by `path`.

diff --git a/tests/unit/repository/test_memory_repo.py b/tests/unit/repository/test_memory_repo.py
@@ -1,4 +1,3 @@
-import os
 import pickle
 
 import fsspec
@@ -15,17 +14,6 @@ def test_initialization():
     assert type(memory_repo.filesystem) == fsspec.implementations.memory.MemoryFileSystem
 
 
-def test_add_dirnames_to_pseudo_dirs():
-    path = "/memory/root/data.json"
-
-    memory_repo = MemoryRepository()
-    memory_repo._add_dirnames_to_pseudo_dirs(path)
-
-    while path != "" and path != "/":
-        path = os.path.dirname(path)
-        assert path in memory_repo.filesystem.pseudo_dirs
-
-
 def test_persist_dataframe():
     path = "/memory/root/data"
     df = pd.DataFrame([[0, 1], [1, 0]], columns=["a", "b"])