Lazy Loading Overhaul

vimiv/api/settings.py

@@ -325,6 +325,7 @@ class OrderSetting(Setting):
         "alphabetical": str,
         "natural": natural_sort,
         "recently-modified": os.path.getmtime,
+		"passthrough": None
     }
 
     STR_ORDER_TYPES = "alphabetical", "natural"
@@ -347,6 +348,8 @@ def convert(self, value: str) -> str:
 
     def sort(self, values: Iterable[str]) -> List[str]:
         """Sort values according to the current ordering."""
+        if self.value == "passthrough":
+            return values
         ordering = self._get_ordering()
         return sorted(values, key=ordering, reverse=sort.reverse.value)
 
@@ -447,6 +450,16 @@ class image:  # pylint: disable=invalid-name
         True,
         desc="Require holding the control modifier for zooming with the mouse wheel",
     )
+    id_by_extension = BoolSetting(
+        "image.id_by_extension",
+        False,
+        desc="Instead of scanning the image to determine that it's a compatible format, assume the extension accurately represents the format.",
+    )
+    imghdr_fallback = BoolSetting(
+        "image.imghdr_fallback",
+        True,
+        desc="If identifying a file by extension (when image.id_by_extension is set) files to open a file, try identifying it using imghdr before giving up."
+    )
 
 
 class library:  # pylint: disable=invalid-name
@@ -469,6 +482,10 @@ class thumbnail:  # pylint: disable=invalid-name
     """Namespace for thumbnail related settings."""
 
     size = ThumbnailSizeSetting("thumbnail.size", 128, desc="Size of thumbnails")
+    save = BoolSetting("thumbnail.save", True, desc="Save thumbnails to disk")
+    max_behind = IntSetting("thumbnail.max_behind", 0, desc="Maximum number of thumbnails to render behind the currently selected one.")
+    max_ahead = IntSetting("thumbnail.max_ahead", 0, desc="Maximum number of thumbnails to render ahead of the currently selected one.")
+    max_count = IntSetting("thumbnail.max_count", 0, desc="Maximum number of thumbnails to render in general.")
 
 
 class slideshow:  # pylint: disable=invalid-name
@@ -610,3 +627,4 @@ class sort:  # pylint: disable=invalid-name
         False,
         desc="Randomly shuffle images and ignoring all other sort settings",
     )
+

vimiv/api/working_directory.py

@@ -61,7 +61,7 @@ def _on_im_changed(self, new_images, added, removed):
 import os
 from typing import cast, List, Tuple
 
-from PyQt5.QtCore import pyqtSignal, QFileSystemWatcher
+from PyQt5.QtCore import pyqtSignal, QFileSystemWatcher, pyqtSlot
 
 from vimiv.api import settings, signals, status
 from vimiv.utils import files, slot, log, throttled
@@ -92,7 +92,8 @@ class WorkingDirectoryHandler(QFileSystemWatcher):
 
     Attributes:
         _dir: The current working directory.
-        _images: Images in the current working directory.
+        _original_images: Originally specified layout of images in the current working directory.
+        _images: images in the current working directory.
         _directories: Directories in the current working directory.
     """
 
@@ -105,6 +106,7 @@ class WorkingDirectoryHandler(QFileSystemWatcher):
     def __init__(self) -> None:
         super().__init__()
         self._dir = ""
+        self._original_images: List[str] = []
         self._images: List[str] = []
         self._directories: List[str] = []
 
@@ -113,6 +115,7 @@ def __init__(self) -> None:
         settings.sort.directory_order.changed.connect(self._reorder_directory)
         settings.sort.reverse.changed.connect(self._reorder_directory)
         settings.sort.ignore_case.changed.connect(self._reorder_directory)
+        signals.load_images.connect(self._load_parameter_images)
 
         self.directoryChanged.connect(self._reload_directory)
         self.fileChanged.connect(self._on_file_changed)
@@ -161,14 +164,18 @@ def _on_monitor_fs_changed(self, value: bool) -> None:
     def _load_directory(self, directory: str) -> None:
         """Load supported files for new directory."""
         self._dir = directory
-        self._images, self._directories = self._get_content(directory)
-        self.loaded.emit(self._images, self._directories)
+        images, directories = self._get_content(directory)
+        self._load_original_images(images)
+        self._images = list(self._original_images)
+        self.loaded.emit(self._original_images, self._directories)
 
     @throttled(delay_ms=WAIT_TIME_MS)
     def _reload_directory(self, _path: str) -> None:
         """Load new supported files when directory content has changed."""
         _logger.debug("Reloading working directory")
-        self._emit_changes(*self._get_content(self._dir))
+        images, directories = self._get_content(self._dir)
+        self._load_original_images(images)
+        self._emit_changes(*self._order_paths(self._original_images, directories))
 
     @slot
     def _on_new_image(self, path: str) -> None:
@@ -216,8 +223,8 @@ def _emit_changes(self, images: List[str], directories: List[str]) -> None:
             self.images_changed.emit(images, added, removed)
         # Total filelist has changed, relevant for the library
         if images != self._images or directories != self._directories:
-            self._images = images
-            self._directories = directories
+            self._images = list(images)
+            self._directories = list(directories)
             self.changed.emit(images, directories)
 
     def _get_content(self, directory: str) -> Tuple[List[str], List[str]]:
@@ -231,11 +238,35 @@ def _get_content(self, directory: str) -> Tuple[List[str], List[str]]:
         paths = files.listdir(directory, show_hidden=show_hidden)
         return self._order_paths(*files.supported(paths))
 
+    @pyqtSlot(list)
+    def _load_parameter_images(self, images: list[str]):
+        old_original_images = self._original_images
+        self._original_images = []
+        self._original_images.extend(images)
+        for i in old_original_images:
+            if i not in self._original_images:
+                self._original_images.append(i)
+        new_original_images = []
+        for i in self._original_images:
+            if i in old_original_images:
+                new_original_images.append(i)
+        self._original_images = new_original_images
+
+    def _load_original_images(self, images: list[str]):
+        for i in images:
+            if i not in self._original_images:
+                self._original_images.append(i)
+        new_original_images = []
+        for i in self._original_images:
+            if i in images:
+                new_original_images.append(i)
+        self._original_images = new_original_images
+
     @slot
     def _reorder_directory(self) -> None:
         """Reorder current files / directories."""
         _logger.debug("Reloading working directory")
-        self._emit_changes(*self._order_paths(self._images, self._directories))
+        self._emit_changes(*self._order_paths(self._original_images, self._directories))
 
     @staticmethod
     def _order_paths(images: List[str], dirs: List[str]) -> Tuple[List[str], List[str]]:

vimiv/gui/thumbnail.py

@@ -83,6 +83,7 @@ def __init__(self) -> None:
         widgets.ScrollWheelCumulativeMixin.__init__(self, self._scroll_wheel_callback)
         QListWidget.__init__(self)
 
+        self._rendered_paths = set()
         self._paths: List[str] = []
 
         fail_pixmap = create_pixmap(
@@ -143,6 +144,69 @@ def n_rows(self) -> int:
         """Return the number of rows."""
         return math.ceil(self.count() / self.n_columns())
 
+    def _first_index(self) -> int:
+        """Return the index of the first thumbnail to be rendered."""
+        result: int
+        if api.settings.thumbnail.max_behind.value == 0:
+            result = 0
+        else:
+            result = max(0,
+                       self.current_index() - api.settings.thumbnail.max_behind.value)
+        return result
+
+    def _last_index(self) -> int:
+        """Return the index of the last thumbnail to be rendered."""
+        result: int
+        if len(self._paths) == 0:
+            result = 0
+        elif api.settings.thumbnail.max_ahead.value == 0:
+            result = len(self._paths) - 1
+        else:
+            result = min(len(self._paths) - 1,
+                       self.current_index() + api.settings.thumbnail.max_ahead.value)
+        return result
+
+    def _index_range(self) -> tuple[int, int]:
+        """Return a tuple with the first index to be rendered, and last, plus one to match a typical Python range."""
+        return (self._first_index(), self._last_index() + 1)
+
+    def _prune_index(self, index) -> None:
+        """Unload the icon associated with a particular path index."""
+        item = self.item(index)
+        # Otherwise it has been deleted in the meanwhile
+        if item is not None and\
+           (api.settings.thumbnail.max_count.value == 0 or\
+           len(self._rendered_paths) > api.settings.thumbnail.max_count.value):
+            item.setIcon(ThumbnailItem.default_icon())
+            self._rendered_paths.discard(self._paths[index])
+
+    def _prune_icons_behind(self) -> None:
+        """Prune indexes before the current index."""
+        for index in range(0, self._first_index()):
+            self._prune_index(index)
+
+    def _prune_icons_ahead(self) -> None:
+        """Prune indexes after the current index."""
+        for index in reversed(range(self._last_index() + 1, len(self._paths))):
+            self._prune_index(index)
+
+    def _prune_icons(self) -> None:
+        """Prune indexes before and after the current index."""
+        self._prune_icons_behind()
+        self._prune_icons_ahead()
+
+    def _update_icon_position(self) -> None:
+        """Adjust displayed range of icons to current position."""
+        first_index, last_index = self._index_range()
+        desired_paths = []
+        indices = []
+        for p, i in zip(self._paths[first_index:last_index], range(first_index, last_index)):
+            if p not in self._rendered_paths:
+                desired_paths.append(p)
+                indices.append(i)
+        self._manager.create_thumbnails_async(indices, desired_paths)
+        self._prune_icons()
+
     def item(self, index: int) -> "ThumbnailItem":
         return cast(ThumbnailItem, super().item(index))
 
@@ -162,6 +226,7 @@ def _on_new_images_opened(self, paths: List[str]):
             _logger.debug("No new images to load")
             return
         _logger.debug("Updating thumbnails...")
+        self._rendered_paths.clear()
         removed = set(self._paths) - set(paths)
         for path in removed:
             _logger.debug("Removing existing thumbnail '%s'", path)
@@ -176,7 +241,9 @@ def _on_new_images_opened(self, paths: List[str]):
                 ThumbnailItem(self, i, size_hint=size_hint)
             self.item(i).marked = path in api.mark.paths  # Ensure correct highlighting
         self._paths = paths
-        self._manager.create_thumbnails_async(paths)
+        first_index, last_index = self._index_range()
+        self._manager.create_thumbnails_async(range(first_index, last_index),
+                                              paths[first_index:last_index])
         _logger.debug("... update completed")
 
     @utils.slot
@@ -190,6 +257,7 @@ def _on_thumbnail_created(self, index: int, icon: QIcon):
         item = self.item(index)
         if item is not None:  # Otherwise it has been deleted in the meanwhile
             item.setIcon(icon)
+            self._rendered_paths.add(self._paths[index])
 
     @pyqtSlot(int, list, api.modes.Mode, bool)
     def _on_new_search(
@@ -238,21 +306,21 @@ def open_selected(self):
         api.signals.load_images.emit([self.current()])
         api.modes.IMAGE.enter()
 
-    @api.keybindings.register("<ctrl>b", "scroll page-up", mode=api.modes.THUMBNAIL)
-    @api.keybindings.register("<ctrl>f", "scroll page-down", mode=api.modes.THUMBNAIL)
+    @api.keybindings.register(("<ctrl>b",  "<page-up>"), "scroll page-up", mode=api.modes.THUMBNAIL)
+    @api.keybindings.register(("<ctrl>f",  "<page-down>"), "scroll page-down", mode=api.modes.THUMBNAIL)
     @api.keybindings.register(
         "<ctrl>u", "scroll half-page-up", mode=api.modes.THUMBNAIL
     )
     @api.keybindings.register(
         "<ctrl>d", "scroll half-page-down", mode=api.modes.THUMBNAIL
     )
-    @api.keybindings.register("k", "scroll up", mode=api.modes.THUMBNAIL)
-    @api.keybindings.register("j", "scroll down", mode=api.modes.THUMBNAIL)
+    @api.keybindings.register(("k", "<up>"), "scroll up", mode=api.modes.THUMBNAIL)
+    @api.keybindings.register(("j", "<down>"), "scroll down", mode=api.modes.THUMBNAIL)
     @api.keybindings.register(
-        ("h", "<button-back>"), "scroll left", mode=api.modes.THUMBNAIL
+        ("h", "<button-back>", "<left>"), "scroll left", mode=api.modes.THUMBNAIL
     )
     @api.keybindings.register(
-        ("l", "<button-forward>"), "scroll right", mode=api.modes.THUMBNAIL
+        ("l", "<button-forward>", "<right>"), "scroll right", mode=api.modes.THUMBNAIL
     )
     @api.commands.register(mode=api.modes.THUMBNAIL)
     def scroll(  # type: ignore[override]
@@ -295,8 +363,8 @@ def _scroll_updown(
             last_in_col -= self.n_columns()
         return min(current + self.n_columns() * step, last_in_col)
 
-    @api.keybindings.register("gg", "goto 1", mode=api.modes.THUMBNAIL)
-    @api.keybindings.register("G", "goto -1", mode=api.modes.THUMBNAIL)
+    @api.keybindings.register(("gg", "<home>"), "goto 1", mode=api.modes.THUMBNAIL)
+    @api.keybindings.register(("G", "<end>"), "goto -1", mode=api.modes.THUMBNAIL)
     @api.commands.register(mode=api.modes.THUMBNAIL)
     def goto(self, index: Optional[int], count: Optional[int] = None):
         """Select specific thumbnail in current filelist.
@@ -375,6 +443,7 @@ def _select_index(self, index: int, emit: bool = True) -> None:
         self.setCurrentRow(index)
         if emit:
             synchronize.signals.new_thumbnail_path_selected.emit(self._paths[index])
+        self._update_icon_position()
 
     def _on_size_changed(self, value: int):
         _logger.debug("Setting size to %d", value)
@@ -446,6 +515,7 @@ def resizeEvent(self, event):
         """Update resize event to keep selected thumbnail centered."""
         super().resizeEvent(event)
         self.scrollTo(self.currentIndex())
+        self._update_icon_position()
 
     def _scroll_wheel_callback(self, steps_x, steps_y):
         """Callback function used by the scroll wheel mixin for mouse scrolling."""

vimiv/utils/files.py

@@ -13,11 +13,26 @@
 
 from PyQt5.QtGui import QImageReader
 
+from vimiv import api
 from vimiv.utils import imagereader
 
 
 ImghdrTestFuncT = Callable[[bytes, Optional[BinaryIO]], bool]
 
+image_format_names = set(('.rgb',
+                          '.gif',
+                          '.pbm',
+                          '.pgm',
+                          '.ppm',
+                          '.tiff',
+                          '.rast',
+                          '.xbm',
+                          '.jpeg',
+                          '.jpg',
+                          '.bmp',
+                          '.png',
+                          '.webp',
+                          '.exr'))
 
 def listdir(directory: str, show_hidden: bool = False) -> List[str]:
     """Wrapper around os.listdir.
@@ -121,7 +136,14 @@ def is_image(filename: str) -> bool:
         filename: Name of file to check.
     """
     try:
-        return os.path.isfile(filename) and imghdr.what(filename) is not None
+        if not os.path.isfile(filename):
+            return False
+        elif api.settings.image.id_by_extension:
+            return os.path.splitext(filename)[1].lower() in image_format_names
+        elif imghdr.what(filename) is not None:
+            return True
+        else:
+            return False
     except OSError:
         return False
 
@@ -164,6 +186,8 @@ def test(h: bytes, f: Optional[BinaryIO]) -> Optional[str]:
             imghdr.tests.remove(test)
         return None
 
+    global image_format_names
+    image_format_names.add(''.join(('.', name)))
     imghdr.tests.insert(add_image_format.index, test)  # type: ignore
     add_image_format.index += 1  # type: ignore
 
@@ -177,7 +201,6 @@ def test_svg(h: bytes, _f: Optional[BinaryIO]) -> bool:
 
 add_image_format("svg", test_svg)
 
-
 def test_ico(h: bytes, _f: Optional[BinaryIO]) -> bool:
     return h.startswith(bytes.fromhex("00000100"))