Updated webp to use multiprocessing

MangaManager/Extensions/WebpConverter/WebpConverter.py

@@ -6,28 +6,18 @@
 import pathlib
 import tkinter
 import tkinter.ttk as ttk
-from concurrent.futures import ProcessPoolExecutor
 from tkinter import filedialog
 
 from Extensions.IExtensionApp import IExtensionApp
-from MangaManager.LoadedComicInfo.LoadedComicInfo import LoadedComicInfo
+from Extensions.WebpConverter.processing import convert
 from MangaManager.Common.utils import ShowPathTreeAsDict
 from MangaManager.MetadataManager.GUI.widgets import ScrolledFrameWidget, ProgressBarWidget
+from MangaManager.ProcessingPool import ProcessingPool
 from MangaManager.Settings.Settings import Settings
 
 logger = logging.getLogger()
 
 
-def convert(file):
-    try:
-        LoadedComicInfo(file, load_default_metadata=False).convert_to_webp()
-    except:
-        logger.exception("Exception converting")
-
-
-
-
-
 class WebpConverter(IExtensionApp):
     name = "Webp Converter"
     embedded_ui = True
@@ -55,7 +45,7 @@ def process(self):
         self._progress_bar.start(len(self._selected_files))
         self._progress_bar.running = True
         self.pb_update()
-        pool = ProcessPoolExecutor()
+        pool = ProcessingPool()
         for file in self._selected_files:
 
             logger.debug(f"[Extension][WebpConvert] Processing file",
@@ -67,7 +57,10 @@ def process(self):
     def done_callback(self,future):
         try:
             result = future.result()
-            self._progress_bar.increase_processed()
+            if result:
+                self._progress_bar.increase_processed()
+            else:
+                self._progress_bar.increase_failed()
         except Exception:
             logger.exception("Exception converting file")
             self._progress_bar.increase_failed()

MangaManager/Extensions/WebpConverter/processing.py

@@ -0,0 +1,14 @@
+import logging
+
+from MangaManager.LoadedComicInfo.LoadedComicInfo import LoadedComicInfo
+
+logger = logging.getLogger()
+
+def convert(file):
+    try:
+        LoadedComicInfo(file, load_default_metadata=False).convert_to_webp()
+        return True
+    except:
+        logger.exception("Exception converting")
+        return False
+

MangaManager/MangaManager/Common/progressbar.py

@@ -4,7 +4,7 @@
 from string import Template
 from threading import Timer
 
-from MangaManager.Common.utils import get_elapsed_time, get_estimated_time
+from .utils import get_elapsed_time, get_estimated_time
 
 logger = logging.getLogger()
 

MangaManager/MangaManager/Common/utils.py

@@ -9,7 +9,7 @@
 from pathlib import Path
 from typing import IO
 
-from MangaManager.Common.naturalsorter import natsort_key_with_path_support
+from .naturalsorter import natsort_key_with_path_support
 
 # Patterns for picking cover
 IMAGE_EXTENSIONS = ('png', 'jpg', 'jpeg', 'tiff', 'bmp', 'gif', 'webp')

MangaManager/MangaManager/LoadedComicInfo/LoadedComicInfo.py

@@ -16,7 +16,6 @@
 from .ILoadedComicInfo import ILoadedComicInfo
 from .LoadedFileCoverData import LoadedFileCoverData
 from .LoadedFileMetadata import LoadedFileMetadata
-from MangaManager.Settings import Settings, SettingHeading
 
 logger = logging.getLogger("LoadedCInfo")
 COMICINFO_FILE = 'ComicInfo.xml'
@@ -116,20 +115,27 @@ def write_metadata(self, auto_unmark_changes=False):
         self.has_changes = self.cinfo_object.has_changes(self.original_cinfo_object)
         logger.debug(f"[{'BEGIN WRITE':13s}] Writing metadata to file '{self.file_path}'")
         try:
-            self._process(write_metadata=self.has_changes)
+            self._process(write_metadata=self.has_changes, do_create_backup_comicinfo=True)
         finally:
             if auto_unmark_changes:
                 self.has_changes = False
 
     def convert_to_webp(self):
         logger.debug(f"[{'BEGIN CONVERT':13s}] Converting to webp: '{self.file_path}'")
-        self._process(do_convert_to_webp=True)
+        self._process(do_convert_to_webp=True, do_create_backup_comicinfo=True)
 
     def _export_metadata(self) -> str:
         return str(self.cinfo_object.to_xml())
 
     # ACTUAL LOGIC
-    def _process(self, write_metadata=False, do_convert_to_webp=False, **_):
+    def _process(self, write_metadata=False, do_convert_to_webp=False, do_create_backup_comicinfo=True, **_):
+        """
+
+        :param write_metadata:
+        :param do_convert_to_webp:
+        :param do_create_backup_comicinfo: Settings().get(SettingHeading.Main, "create_backup_comicinfo") == 'True'
+        :return:
+        """
         logger.info(f"[{'PROCESSING':13s}] Processing file '{self.file_path}'")
         if write_metadata and not do_convert_to_webp and not self.has_metadata:
             with zipfile.ZipFile(self.file_path, mode='a', compression=zipfile.ZIP_STORED) as zf:
@@ -152,7 +158,7 @@ def _process(self, write_metadata=False, do_convert_to_webp=False, **_):
                     orig_comp_type = s.compress_type
                     break
             with zipfile.ZipFile(tmpname, "w",compression=orig_comp_type) as zout:  # The temp file where changes will be saved to
-                self._recompress(zin, zout, write_metadata=write_metadata, do_convert_webp=do_convert_to_webp)
+                self._recompress(zin, zout, write_metadata=write_metadata, do_convert_webp=do_convert_to_webp,do_create_backup_comicinfo=False)
             newfile_size = os.path.getsize(tmpname)
 
             # If the new file is smaller than the original file, we process again with no webp conversion.
@@ -166,7 +172,7 @@ def _process(self, write_metadata=False, do_convert_to_webp=False, **_):
                     return
                 logger.warning(f"[{'Processing':13s}]  ⤷ Cover action or new metadata detected. Processing new covers without converting source to webp")
                 with zipfile.ZipFile(tmpname, "w") as zout:  # The temp file where changes will be saved to
-                    self._recompress(zin, zout, write_metadata=write_metadata, do_convert_webp=False)
+                    self._recompress(zin, zout, write_metadata=write_metadata, do_convert_webp=False,do_create_backup_comicinfo=do_create_backup_comicinfo)
 
         # Reset cover flags
         self.cover_action = CoverActions.RESET
@@ -211,7 +217,7 @@ def _process(self, write_metadata=False, do_convert_to_webp=False, **_):
             logger.info("[{'Processing':13s}] Updating covers")
             self.load_cover_info()
 
-    def _recompress(self, zin, zout, write_metadata, do_convert_webp):
+    def _recompress(self, zin, zout, write_metadata, do_convert_webp,do_create_backup_comicinfo):
         """
         Given 2 input and output zipfiles copy content of one zipfile to the new one.
         Files that matches certain criteria gets skipped and not copied over, hence deleted.
@@ -230,7 +236,7 @@ def _recompress(self, zin, zout, write_metadata, do_convert_webp):
             logger.debug(f"[{_LOG_TAG_WRITE_META:13s}] New ComicInfo.xml appended to the file")
             # Directly backup the metadata if it's at root.
             if self.is_cinfo_at_root:
-                if Settings().get(SettingHeading.Main, "create_backup_comicinfo") == 'True' and self.had_metadata_on_open:
+                if do_create_backup_comicinfo and self.had_metadata_on_open:
                     zout.writestr(f"Old_{COMICINFO_FILE}.bak", zin.read(COMICINFO_FILE))
                     logger.debug(f"[{_LOG_TAG_WRITE_META:13s}] Backup for comicinfo.xml created")
                 is_metadata_backed = True
@@ -264,7 +270,7 @@ def _recompress(self, zin, zout, write_metadata, do_convert_webp):
                         continue
 
                     # If filename is comicinfo save as old_comicinfo.xml
-                    if Settings().get(SettingHeading.Main, "create_backup_comicinfo") == 'True' and self.had_metadata_on_open:
+                    if do_create_backup_comicinfo and self.had_metadata_on_open:
                         zout.writestr(f"Old_{item.filename}.bak", zin.read(item.filename))
                         logger.debug(f"[{_LOG_TAG_WRITE_META:13s}] Backup for comicinfo.xml created")
                     # Stop accepting more comicinfo files.

MangaManager/MangaManager/LoadedComicInfo/LoadedFileCoverData.py

@@ -12,7 +12,6 @@
 from .ArchiveFile import ArchiveFile
 from .CoverActions import CoverActions
 from .ILoadedComicInfo import ILoadedComicInfo
-from MangaManager.Settings import Settings, SettingHeading
 
 logger = logging.getLogger("LoadedCInfo")
 COMICINFO_FILE = 'ComicInfo.xml'
@@ -112,7 +111,13 @@ def new_backcover_path(self, path):
             self.new_cover_cache = None
         self._new_backcover_path = path
 
-    def load_cover_info(self, load_images=True):
+    def load_cover_info(self, load_images=True,cache_cover_images=True):
+        """
+
+        :param load_images:
+        :param cache_cover_images: bool(Settings().get(SettingHeading.Main, 'cache_cover_images'))
+        :return:
+        """
         try:
             with ArchiveFile(self.file_path,'r') as self.archive:
                 cover_info = obtain_cover_filename(self.archive.namelist())
@@ -124,7 +129,7 @@ def load_cover_info(self, load_images=True):
                     logger.warning(f"[{'CoverParsing':13s}] Couldn't parse any cover")
                 else:
                     logger.info(f"[{'CoverParsing':13s}] Cover parsed as '{self.cover_filename}'")
-                    if bool(Settings().get(SettingHeading.Main, 'cache_cover_images')):
+                    if cache_cover_images:
                         self.get_cover_image_bytes()
 
                 if not self.backcover_filename:

MangaManager/MangaManager/LoadedComicInfo/LoadedFileMetadata.py

@@ -5,7 +5,7 @@
 from lxml.etree import XMLSyntaxError
 
 from ComicInfo import ComicInfo
-from MangaManager.LoadedComicInfo.ILoadedComicInfo import ILoadedComicInfo
+from .ILoadedComicInfo import ILoadedComicInfo
 from MangaManager.Common.errors import MissingRarTool
 
 logger = logging.getLogger("LoadedCInfo")

MangaManager/MangaManager/ProcessingPool.py

@@ -0,0 +1,14 @@
+from concurrent.futures import ProcessPoolExecutor
+
+
+class ProcessingPool(object):
+    _instance = None
+
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            cls._instance = object.__new__(cls)
+            cls._instance.pool = ProcessPoolExecutor()
+        return cls._instance
+
+    def submit(self, func, *args, **kwargs):
+        return self.pool.submit(func, *args, **kwargs)

MangaManager/MangaManager/__init__.py

@@ -24,4 +24,3 @@
 SOURCES_DIR.mkdir(exist_ok=True)
 
 loaded_extensions =  []
-

MangaManager/logging_setup.py

@@ -37,7 +37,7 @@ def setup_logging(LOGFILE_PATH,level=logging.DEBUG):
     stream_handler = logging.StreamHandler(sys.stdout)
     stream_handler.setLevel(level)
     logging.basicConfig(level=logging.DEBUG,
-                        format='%(asctime)s - %(name)20s - %(levelname)8s - %(message)s',
+                        format='%(asctime)s - [%(process)5s-%(thread)5s] - %(name)20s - %(levelname)8s - %(message)s',
                         handlers=[stream_handler, rotating_file_handler, umpumped_handler]
                         # filename='/tmp/myapp.log'
                         )

MangaManager/main.py

@@ -43,38 +43,39 @@
 setup_logging(LOGFILE_PATH, args.loglevel)
 logger = logging.getLogger()
 
-from MangaManager.Settings.Settings import Settings
-# Create initial ini with defaults else load existing
-Settings().load()
-from MangaManager.Common.errors import NoFilesSelected
-from MangaManager.MetadataManager.MetadataManagerCLI import App as CLIMetadataApp
-from MangaManager.__version__ import __version__ as version
+if __name__ == '__main__':
+    from MangaManager.Settings.Settings import Settings
+    # Create initial ini with defaults else load existing
+    Settings().load()
+    from MangaManager.Common.errors import NoFilesSelected
+    from MangaManager.MetadataManager.MetadataManagerCLI import App as CLIMetadataApp
+    from MangaManager.__version__ import __version__ as version
 
 
 
-# <Arguments parser>
+    # <Arguments parser>
 
 
-class ToolS(enum.Enum):
-    NONE = 0
-    METADATA = 1
-    WEBP = 5
+    class ToolS(enum.Enum):
+        NONE = 0
+        METADATA = 1
+        WEBP = 5
 
-    @classmethod
-    def list(cls):
-        return list(map(lambda c: c.name, cls))
+        @classmethod
+        def list(cls):
+            return list(map(lambda c: c.name, cls))
 
 
-def get_selected_files(glob_path) -> list[str]:
-    file_paths = glob.glob(glob_path)
-    if not file_paths:
-        raise NoFilesSelected()
-    return file_paths
+    def get_selected_files(glob_path) -> list[str]:
+        file_paths = glob.glob(glob_path)
+        if not file_paths:
+            raise NoFilesSelected()
+        return file_paths
+
 
 
 
 
-if __name__ == '__main__':
     if args.selected_files_cli:
         logger.info(f"Starting: CLI Metadata app")
         selected_files = get_selected_files(args.selected_files_cli)