Run all pyvips calls in process pools

DIAGNijmegen · Jan 23, 2025 · 3b4ed9c · 3b4ed9c
1 parent 73de7ca
commit 3b4ed9c
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 7 deletions.
diff --git a/panimg/image_builders/tiff.py b/panimg/image_builders/tiff.py
@@ -2,6 +2,7 @@
 import re
 from collections import defaultdict
 from collections.abc import Callable, Iterator
+from concurrent.futures import ProcessPoolExecutor
 from dataclasses import dataclass, field
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -320,11 +321,13 @@ def _convert(
             if associated_files_getter:
                 associated_files = associated_files_getter(gc_file.path)
 
-            tiff_file = _convert_to_tiff(
-                path=file,
-                pk=gc_file.pk,
-                output_directory=output_directory,
-            )
+            with ProcessPoolExecutor(max_workers=1) as executor:
+                tiff_file = executor.submit(
+                    _convert_to_tiff,
+                    path=file,
+                    pk=gc_file.pk,
+                    output_directory=output_directory,
+                ).result()
         except Exception as e:
             file_errors[file].append(
                 format_error(
@@ -529,7 +532,10 @@ def image_builder_tiff(  # noqa: C901
 
             # try and load image with open slide
             try:
-                gc_file = _load_with_openslide(gc_file=gc_file)
+                with ProcessPoolExecutor(max_workers=1) as executor:
+                    gc_file = executor.submit(
+                        _load_with_openslide, gc_file=gc_file
+                    ).result()
             except Exception:
                 file_errors[gc_file.path].append(
                     format_error("Could not open file with OpenSlide.")

diff --git a/panimg/post_processors/tiff_to_dzi.py b/panimg/post_processors/tiff_to_dzi.py
@@ -1,4 +1,5 @@
 import logging
+from concurrent.futures import ProcessPoolExecutor
 
 from panimg.models import ImageType, PanImgFile, PostProcessorResult
 from panimg.settings import DZI_TILE_SIZE
@@ -12,7 +13,10 @@ def tiff_to_dzi(*, image_files: set[PanImgFile]) -> PostProcessorResult:
     for file in image_files:
         if file.image_type == ImageType.TIFF:
             try:
-                result = _create_dzi_image(tiff_file=file)
+                with ProcessPoolExecutor(max_workers=1) as executor:
+                    result = executor.submit(
+                        _create_dzi_image, tiff_file=file
+                    ).result()
             except Exception as e:
                 logger.warning(f"Could not create DZI for {file}: {e}")
                 continue