Feature/tfaehse#37 blur mask (tfaehse#41)

 - remove two stage blurring, use two masks instead
- add feathered mask
- add mask edge feather argument
- change cli bounds and default value formatting
- add changed cli to readme
closes tfaehse#37
- add cli option to export mask only
closes tfaehse#40

README.md

@@ -97,30 +97,40 @@ For reference: even at 1080p inference, i.e. an inference scale of 1, a 1080p30f
 There's now also a fairly simple CLI to blur a video:
 
 ```
-python cli.py -h
-usage: cli.py [-h] -i INPUT -o OUTPUT [-w WEIGHTS] [-s [1,1024]] [-b [1-99]] [-if [144-2160]]
-              [-t [0-1]] [-r [0-2]] [-q [1, 10]] -f [0-5] [-nf]
+usage: cli.py [-h] -i INPUT -o OUTPUT [-w WEIGHTS] [-s [1, 1024] = 1] [-b [1, 99] = 9]
+              [-if [144, 2160] = 720] [-t [0.0, 1.0] = 0.4] [-r [0.0, 2.0] = 1.0] [-q [1.0, 10.0] =
+              10.0] [-f [0, 5] = 0] [-fe [0, 99] = 5] [-nf]
 
 This tool allows you to automatically censor faces and number plates on dashcam footage.
 
-options:
-  -h, --help                        show this help message and exit
+optional arguments:
+  -h, --help                            show this help message and exit
 
 required arguments:
-  -i, --input INPUT                 input video file path
-  -o, --output OUTPUT               output video file path
+  -i, --input INPUT                     Input video file path.
+  -o, --output OUTPUT                   Output video file path.
 
 optional arguments:
-  -w, --weights WEIGHTS             Weights file to use. See readme for the differences
-  -s, --batch_size [1,1024]         inference batch size - large values require a lof of memory and
-                                    may cause crashes!
-  -b, --blur_size [1-99]            granularity of the blurring filter
-  -if, --inference_size [144-2160]  vertical inference size, e.g. 1080 or 720
-  -t, --threshold [0-1]             detection threshold
-  -r, --roi_multi [0-2]             increase/decrease area that will be blurred - 1 means no change
-  -q, --quality [1, 10]             quality of the resulting video. higher = better, default: 10
-  -f, --frame_memory [0-5]          blur objects in the last x frames too
-  -nf, --no_faces                   do not censor faces
+  -w, --weights WEIGHTS                 Weights file to use. See readme for the differences.
+                                        (default = 720p_medium_mosaic).
+  -s, --batch_size [1, 1024] = 1        Inference batch size - large values require a lof of memory
+                                        and may cause crashes! Not recommended for CPU usage.
+  -b, --blur_size [1, 99] = 9           Kernel radius of the gauss-filter.
+  -if, --inference_size [144, 2160] = 720
+                                        Vertical inference size, e.g. 1080 or 720.
+  -t, --threshold [0.0, 1.0] = 0.4      Detection threshold. Higher value means more certainty,
+                                        lower value means more blurring.
+  -r, --roi_multi [0.0, 2.0] = 1.0      Increase/decrease area that will be blurred - 1.0 means no
+                                        change.
+  -q, --quality [1.0, 10.0] = 10.0      Quality of the resulting video. higher = better. conversion
+                                        to crf: ⌊(1-q/10)*51⌋.
+  -f, --frame_memory [0, 5] = 0         Blur objects in the last x frames too.
+  -fe, --feather_edges [0, 99] = 5      Feather edges of blurred areas, removes sharp edges on blur-
+                                        mask. expands mask by argument and blurs mask, so effective
+                                        size is twice the argument.
+  -nf, --no_faces                       Fo not censor faces.
+  -m, --export_mask                     Export a black and white only video of the blur-mask without
+                                        applying it to the input clip.
 ```
 For now, there are no default values and all parameters have to be provided (in order). There's also no progress bar yet, but there should be an error/success message as soon as blurring is finished/has encountered any issues.
 

dashcamcleaner/cli.py

@@ -33,6 +33,8 @@ def start_blurring(self):
             "quality": self.opt.quality,
             "batch_size": self.opt.batch_size,
             "no_faces": self.opt.no_faces,
+            "feather_edges": self.opt.feather_edges,
+            "export_mask": self.opt.export_mask,
         }
 
         # setup blurrer
@@ -60,12 +62,12 @@ def _format_action_invocation(self, action):
     )
 
     required = parser.add_argument_group("required arguments")
-    required.add_argument("-i", "--input", required=True, help="input video file path", type=str)
+    required.add_argument("-i", "--input", required=True, help="Input video file path.", type=str)
     required.add_argument(
         "-o",
         "--output",
         required=True,
-        help="output video file path",
+        help="Output video file path.",
         type=str,
     )
 
@@ -74,79 +76,97 @@ def _format_action_invocation(self, action):
         "-w",
         "--weights",
         required=False,
-        default="720p_medium_mosaic",
-        help="Weights file to use. See readme for the differences",
+        help="Weights file to use. See readme for the differences. (default = 720p_medium_mosaic).",
         type=str,
+        default="720p_medium_mosaic",
     )
     optional.add_argument(
         "-s",
         "--batch_size",
-        help="inference batch size - large values require a lof of memory and may cause crashes!",
+        help="Inference batch size - large values require a lof of memory and may cause crashes! Not recommended for CPU usage.",
         type=int,
+        metavar="[1, 1024] = 1",
         default=1,
-        metavar="[1,1024]",
     )
     optional.add_argument(
         "-b",
         "--blur_size",
         required=False,
-        help="granularity of the blurring filter",
+        help="Kernel radius of the gauss-filter.",
         type=int,
+        metavar="[1, 99] = 9",
         default=9,
-        metavar="[1-99]",
     )
     optional.add_argument(
         "-if",
         "--inference_size",
-        help="vertical inference size, e.g. 1080 or 720",
+        help="Vertical inference size, e.g. 1080 or 720.",
         type=int,
+        metavar="[144, 2160] = 720",
         default=720,
-        metavar="[144-2160]",
     )
     optional.add_argument(
         "-t",
         "--threshold",
         required=False,
-        help="detection threshold",
+        help="Detection threshold. Higher value means more certainty, lower value means more blurring.",
         type=float,
+        metavar="[0.0, 1.0] = 0.4",
         default=0.4,
-        metavar="[0-1]",
     )
     optional.add_argument(
         "-r",
         "--roi_multi",
         required=False,
-        help="increase/decrease area that will be blurred - 1 means no change",
+        help="Increase/decrease area that will be blurred - 1.0 means no change.",
         type=float,
+        metavar="[0.0, 2.0] = 1.0",
         default=1.0,
-        metavar="[0-2]",
     )
     optional.add_argument(
         "-q",
         "--quality",
-        metavar="[1, 10]",
         required=False,
-        help="quality of the resulting video. higher = better, default: 10. conversion to crf: ⌊(1-q/10)*51⌋",
+        help="Quality of the resulting video. higher = better. conversion to crf: ⌊(1-q/10)*51⌋.",
         type=float,
         choices=[round(x / 10, ndigits=2) for x in range(10, 101)],
+        metavar="[1.0, 10.0] = 10.0",
         default=10,
     )
     optional.add_argument(
         "-f",
         "--frame_memory",
         required=False,
-        help="blur objects in the last x frames too",
+        help="Blur objects in the last x frames too.",
         type=int,
-        metavar="[0-5]",
+        metavar="[0, 5] = 0",
         choices=range(5 + 1),
         default=0,
     )
+    optional.add_argument(
+        "-fe",
+        "--feather_edges",
+        required=False,
+        help="Feather edges of blurred areas, removes sharp edges on blur-mask. expands mask by argument and blurs mask, so effective size is twice the argument.",
+        type=int,
+        metavar="[0, 99] = 5",
+        choices=range(99 + 1),
+        default=5,
+    )
     optional.add_argument(
         "-nf",
         "--no_faces",
         action="store_true",
         required=False,
-        help="do not censor faces",
+        help="Do not censor faces.",
+        default=False,
+    )
+    optional.add_argument(
+        "-m",
+        "--export_mask",
+        action="store_true",
+        required=False,
+        help="Export a black and white only video of the blur-mask without applying it to the input clip.",
         default=False,
     )
     return parser.parse_args()

dashcamcleaner/src/blurrer.py

@@ -46,6 +46,8 @@ def apply_blur(self: 'VideoBlurrer', frame: np.array, new_detections: List[Detec
         blur_memory = self.parameters["blur_memory"]
         roi_multi = self.parameters["roi_multi"]
         no_faces = self.parameters["no_faces"]
+        feather_dilate_size = self.parameters["feather_edges"]
+        export_mask = self.parameters["export_mask"]
 
         # gather and process all currently relevant detections
         self.detections = [
@@ -63,55 +65,52 @@ def apply_blur(self: 'VideoBlurrer', frame: np.array, new_detections: List[Detec
             # there are no detections for this frame, leave early and return the same input-frame
             return frame
 
-        # prepare copy and mask
-        blur_1 = cv2.GaussianBlur(frame, (blur_size, blur_size), 0)
-        # blurring again with the same kernel is the same as blurring with a kernel sqrt(2) as big (but should be faster)
-        # the variance of a kernel with size N is N^2
-        # the variance of a kernel with size 2*N is 4*N^2
-        # so the necessary variance to add on-top is 3 * N^2, whose root is sqrt(3) * N
-        # which is why the size of the second blur is sqrt(3) * N to get double the gaussion blur size
-        # but with a smaller kernel, since we already have a blurred image with kernel size=N usable as input
-        #
-        # from Wikipedia https://en.wikipedia.org/wiki/Gaussian_blur:
-        # > Applying successive Gaussian blurs to an image has the same effect as applying a single, larger Gaussian blur,
-        # whose radius is the square root of the sum of the squares of the blur radii that were actually applied.
-        second_blur_size = int((blur_size * sqrt(3)) // 2 * 2 + 1)  # has to be odd for the Gaussian blur, so ceil or floor are not usable
-        blur_2 = cv2.GaussianBlur(blur_1, (second_blur_size, second_blur_size), 0)
-
-        mask_blur_1 = np.full((frame.shape[0], frame.shape[1], 1), 0, dtype=np.uint8)
-        mask_blur_2 = np.full((frame.shape[0], frame.shape[1], 1), 0, dtype=np.uint8)
-
-        inner_factor = 0.8
+        # convert to float, since the mask needs to be in range [0, 1] and in float
+        frame = np.float64(frame)
+
+        # prepare mask
+        blur_mask = np.full((frame.shape[0], frame.shape[1], 3), 0, dtype=np.float64)
+        blur_mask_expanded = np.full((frame.shape[0], frame.shape[1], 3), 0, dtype=np.float64)
+
+        if export_mask:
+            mask_color = (255, 255, 255)
+        else:
+            mask_color = (1, 1, 1)
+
         for detection in self.detections:
-            # two-fold blurring: softer blur on the edge of the box to look smoother and less abrupt
-            outer_box = detection.bounds
-            inner_box = detection.bounds.scale(frame.shape, inner_factor)
-
-            if detection.kind == "plate":
-                cv2.rectangle(mask_blur_1, outer_box.pt1(), outer_box.pt2(), color=(255, 255, 255), thickness=-1)
-                cv2.rectangle(mask_blur_2, inner_box.pt1(), inner_box.pt2(), color=(255, 255, 255), thickness=-1)
-            elif detection.kind == "face":
-                center_outer, axes_outer = outer_box.ellipse_coordinates()
-                center_inner, axes_inner = inner_box.ellipse_coordinates()
-                # add ellipse to mask
-                cv2.ellipse(mask_blur_1, center_inner, axes_outer, 0, 0, 360, (255, 255, 255), -1)
-                cv2.ellipse(mask_blur_2, center_outer, axes_inner, 0, 0, 360, (255, 255, 255), -1)
-            else:
-                raise ValueError(f"Detection kind not supported: {detection.kind}")
-
-        # apply mask to blur
-        mask_background = cv2.bitwise_not(cv2.bitwise_or(mask_blur_1, mask_blur_2))
-
-        # remove second blur-mask from first blur-mask,
-        # so as not to add the second (smaller) blur mask twice to the output
-        # https://en.wikipedia.org/wiki/Material_nonimplication
-        mask_blur_1 = cv2.bitwise_and(mask_blur_1, cv2.bitwise_not(mask_blur_2))
-
-        background = cv2.bitwise_and(frame, frame, mask=mask_background)
-        blurred_1 = cv2.bitwise_and(blur_1, blur_1, mask=mask_blur_1)
-        blurred_2 = cv2.bitwise_and(blur_2, blur_2, mask=mask_blur_2)
-        blurred = cv2.add(blurred_1, blurred_2)
-        return cv2.add(background, blurred)
+            bounds_list = [detection.bounds]
+            mask_list = [blur_mask]
+            if feather_dilate_size > 0:
+                bounds_list.append(detection.bounds.expand(frame.shape, feather_dilate_size))
+                mask_list.append(blur_mask_expanded)
+
+            for bounds, mask in zip(bounds_list, mask_list):
+                # add detection bounds to mask
+                if detection.kind == "plate":
+                    cv2.rectangle(mask, bounds.pt1(), bounds.pt2(), color=mask_color, thickness=-1)
+                elif detection.kind == "face":
+                    center, axes = bounds.ellipse_coordinates()
+                    # add ellipse to mask
+                    cv2.ellipse(mask, center, axes, 0, 0, 360, color=mask_color, thickness=-1)
+                else:
+                    raise ValueError(f"Detection kind not supported: {detection.kind}")
+
+        if feather_dilate_size > 0:
+            # blur mask, to feather its edges
+            feather_size = (feather_dilate_size * 3) // 2 * 2 + 1
+            blur_mask_feathered = cv2.GaussianBlur(blur_mask_expanded, (feather_size, feather_size), 0)
+            blur_mask = cv2.min(cv2.add(blur_mask, blur_mask_feathered), mask_color)  # do not oversaturate blurred regions, limit mask to max-value of 1 (for all three channels)
+
+        if export_mask:
+            return np.uint8(blur_mask)
+
+        # to get the background, invert the blur_mask, i.e. 1 - mask on a matrix per-element level
+        mask_background = cv2.subtract(np.full((frame.shape[0], frame.shape[1], 3), mask_color[0], dtype=np.float64), blur_mask)
+
+        background = cv2.multiply(frame, mask_background)
+        blur = cv2.GaussianBlur(frame, (blur_size, blur_size), 0)
+        blurred = cv2.multiply(blur, blur_mask)
+        return np.uint8(cv2.add(background, blurred))
 
     def detect_identifiable_information(self: 'VideoBlurrer', images: list) -> List[List[Detection]]:
         """

dashcamcleaner/src/bounds.py

@@ -36,7 +36,18 @@ def pt1(self: 'Bounds') -> Tuple[int, int]:
     def pt2(self: 'Bounds') -> Tuple[int, int]:
         return (self.x_min, self.y_min)
 
-    def scale(self: 'Bounds', shape, multiplier):
+    def expand(self: 'Bounds', shape, amount: int) -> 'Bounds':
+        frame_height, frame_width = shape[:2]
+
+        scaled_detection = Bounds(
+            max(self.x_min - amount, 0),
+            max(self.y_min - amount, 0),
+            min(self.x_max + amount, frame_width),
+            min(self.y_max + amount, frame_height),
+        )
+        return scaled_detection
+
+    def scale(self: 'Bounds', shape, multiplier) -> 'Bounds':
         """
         Scales a bounding box by a size multiplier and while respecting image dimensions
         :param shape: shape of the image