SensitiveDataSerializer bug fix

Btms.Backend.Cli/Features/RedactImportNotifications/RedactImportNotificationsCommand.cs

@@ -0,0 +1,53 @@
+using System.IO.Compression;
+using Amazon.Runtime.Internal.Util;
+using Btms.Backend.Cli.Features.DownloadScenarioData;
+using Btms.Business.Commands;
+using Btms.Model.Ipaffs;
+using Btms.SensitiveData;
+using CommandLine;
+using MediatR;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Refit;
+using ImportNotification = Btms.Types.Ipaffs.ImportNotification;
+
+namespace Btms.Backend.Cli.Features.RedactImportNotifications;
+
+[Verb("redact-import-notifications", isDefault: false, HelpText = "Redacts Import Notification files.")]
+internal class RedactImportNotificationsCommand : IRequest
+{
+    [Option('r', "rootFolder", Required = true, HelpText = "The root folder to search within")]
+    public required string RootFolder { get; set; }
+
+    public class Handler(ILogger<RedactImportNotificationsCommand> logger) : IRequestHandler<RedactImportNotificationsCommand>
+    {
+        public async Task Handle(RedactImportNotificationsCommand request, CancellationToken cancellationToken)
+        {
+            DirectoryInfo di =
+                new DirectoryInfo(request.RootFolder);
+
+            var files = di.GetFiles("*.json", SearchOption.AllDirectories);
+
+            logger.LogInformation("Found {Count} files", files.Length);
+
+            await Parallel.ForEachAsync(files, cancellationToken, async (fileInfo, ct) =>
+            {
+                logger.LogInformation("Starting file {File}", fileInfo.FullName);
+                var json = await File.ReadAllTextAsync(fileInfo.FullName, ct);
+
+                var options = new SensitiveDataOptions { Include = false };
+                var serializer =
+                    new SensitiveDataSerializer(Options.Create(options), NullLogger<SensitiveDataSerializer>.Instance);
+
+                var result = serializer.RedactRawJson(json, typeof(ImportNotification));
+                await File.WriteAllTextAsync(fileInfo.FullName, result, ct);
+
+                logger.LogInformation("Completed file {File}", fileInfo.FullName);
+            });
+
+        }
+    }
+
+
+}

Btms.SensitiveData.Tests/SensitiveDataSerializerTests.cs

@@ -20,7 +20,8 @@ public void WhenDoNotIncludeSensitiveData_ThenDataShouldBeRedacted()
             SimpleStringOne = "Test String One",
             SimpleStringTwo = "Test String Two",
             SimpleStringArrayOne = ["Test String Array One Item One", "Test String Array One Item Two"],
-            SimpleStringArrayTwo = ["Test String Array Two Item One", "Test String Array Two Item Two"]
+            SimpleStringArrayTwo = ["Test String Array Two Item One", "Test String Array Two Item Two"],
+            SimpleObjectArray = [new SimpleInnerClass() { SimpleStringOne = "Test Inner String" }]
         };
 
         var json = JsonSerializer.Serialize(simpleClass);
@@ -35,6 +36,7 @@ public void WhenDoNotIncludeSensitiveData_ThenDataShouldBeRedacted()
         result.SimpleStringArrayOne[1].Should().Be("TestRedacted");
         result.SimpleStringArrayTwo[0].Should().Be("Test String Array Two Item One");
         result.SimpleStringArrayTwo[1].Should().Be("Test String Array Two Item Two");
+        result.SimpleObjectArray[0].SimpleStringOne.Should().Be("TestRedacted");
     }
 
     [Fact]
@@ -80,7 +82,8 @@ public void WhenDoNotIncludeSensitiveData_AndRequestForRawJson_ThenDataShouldBeR
             SimpleStringTwo = "Test String Two",
             SimpleStringArrayOne =
                 ["Test String Array One Item One", "Test String Array One Item Two"],
-            SimpleStringArrayTwo = ["Test String Array Two Item One", "Test String Array Two Item Two"]
+            SimpleStringArrayTwo = ["Test String Array Two Item One", "Test String Array Two Item Two"],
+            SimpleObjectArray = [new SimpleInnerClass() { SimpleStringOne = "Test Inner String" }]
         };
 
         var json = JsonSerializer.Serialize(simpleClass, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase });
@@ -96,5 +99,6 @@ public void WhenDoNotIncludeSensitiveData_AndRequestForRawJson_ThenDataShouldBeR
         resultClass?.SimpleStringArrayOne[1].Should().Be("TestRedacted");
         resultClass?.SimpleStringArrayTwo[0].Should().Be("Test String Array Two Item One");
         resultClass?.SimpleStringArrayTwo[1].Should().Be("Test String Array Two Item Two");
+        resultClass?.SimpleObjectArray[0].SimpleStringOne.Should().Be("TestRedacted");
     }
 }

Btms.SensitiveData.Tests/SimpleClass.cs

@@ -9,4 +9,13 @@ public class SimpleClass
     [SensitiveData] public string[] SimpleStringArrayOne { get; set; } = null!;
 
     public string[] SimpleStringArrayTwo { get; set; } = null!;
+
+
+    public SimpleInnerClass[] SimpleObjectArray { get; set; } = null!;
+}
+
+
+public class SimpleInnerClass
+{
+    [SensitiveData] public string SimpleStringOne { get; set; } = null!;
 }

Btms.SensitiveData/SensitiveDataSerializer.cs

@@ -1,6 +1,7 @@
 using System.Text.Json;
 using System.Text.Json.Nodes;
 using System.Text.Json.Serialization;
+using System.Text.RegularExpressions;
 using Btms.Common.Extensions;
 using Json.Patch;
 using Json.Path;
@@ -62,39 +63,77 @@ public string RedactRawJson(string json, Type type)
 
         var rootNode = JsonNode.Parse(json);
 
-        foreach (var sensitiveField in sensitiveFields)
+        var jsonPaths = EnumeratePaths(json).ToList();
+        var regex = new Regex("\\[\\d\\]", RegexOptions.Compiled, TimeSpan.FromSeconds(2));
+        foreach (var path in jsonPaths)
         {
-            var jsonPath = JsonPath.Parse($"$.{sensitiveField}");
-            var result = jsonPath.Evaluate(rootNode);
+            var pathStripped = regex.Replace(path, "");
 
-            foreach (var match in result.Matches)
+            if (sensitiveFields.Contains(pathStripped))
             {
-                JsonPatch patch;
-                if (match.Value is JsonArray jsonArray)
-                {
-                    var redactedList = jsonArray.Select(x =>
-                    {
-                        var redactedValue = options.Value.Getter(x?.GetValue<string>()!);
-                        return redactedValue;
-                    }).ToJson();
+                var jsonPath = JsonPath.Parse($"$.{path}");
+                var result = jsonPath.Evaluate(rootNode);
 
-                    patch = new JsonPatch(PatchOperation.Replace(JsonPointer.Parse($"{match.Location!.AsJsonPointer()}"), JsonNode.Parse(redactedList)));
-                }
-                else
+                foreach (var match in result.Matches)
                 {
                     var redactedValue = options.Value.Getter(match.Value?.GetValue<string>()!);
-                    patch = new JsonPatch(PatchOperation.Replace(JsonPointer.Parse(match.Location!.AsJsonPointer()), redactedValue));
+                    var patch = new JsonPatch(PatchOperation.Replace(JsonPointer.Parse(match.Location!.AsJsonPointer()),
+                        redactedValue));
+
+                    var patchResult = patch.Apply(rootNode);
+                    if (patchResult.IsSuccess)
+                    {
+                        rootNode = patchResult.Result;
+                    }
                 }
+            }
+        }
 
+        return rootNode!.ToJsonString(new JsonSerializerOptions() { WriteIndented = true });
+    }
 
-                var patchResult = patch.Apply(rootNode);
-                if (patchResult.IsSuccess)
-                {
-                    rootNode = patchResult.Result;
-                }
+    private static IEnumerable<string> EnumeratePaths(string json)
+    {
+        var doc = JsonDocument.Parse(json).RootElement;
+        var queue = new Queue<(string ParentPath, JsonElement element)>();
+        queue.Enqueue(("", doc));
+        while (queue.Any())
+        {
+            var (parentPath, element) = queue.Dequeue();
+            foreach (var v in QueueIterator(element, parentPath, queue))
+            {
+                yield return v;
             }
         }
+    }
 
-        return rootNode!.ToJsonString();
+    private static IEnumerable<string> QueueIterator(JsonElement element, string parentPath, Queue<(string ParentPath, JsonElement element)> queue)
+    {
+        switch (element.ValueKind)
+        {
+            case JsonValueKind.Object:
+                parentPath = parentPath == ""
+                    ? parentPath
+                    : parentPath + ".";
+                foreach (var nextEl in element.EnumerateObject())
+                {
+                    queue.Enqueue(($"{parentPath}{nextEl.Name}", nextEl.Value));
+                }
+                break;
+            case JsonValueKind.Array:
+                foreach (var (nextEl, i) in element.EnumerateArray().Select((jsonElement, i) => (jsonElement, i)))
+                {
+                    queue.Enqueue(($"{parentPath}[{i}]", nextEl));
+                }
+                break;
+            case JsonValueKind.Undefined:
+            case JsonValueKind.String:
+            case JsonValueKind.Number:
+            case JsonValueKind.True:
+            case JsonValueKind.False:
+            case JsonValueKind.Null:
+                yield return parentPath;
+                break;
+        }
     }
 }