diff --git a/Btms.Backend.Cli/Features/RedactImportNotifications/RedactImportNotificationsCommand.cs b/Btms.Backend.Cli/Features/RedactImportNotifications/RedactImportNotificationsCommand.cs new file mode 100644 index 00000000..fa290e2c --- /dev/null +++ b/Btms.Backend.Cli/Features/RedactImportNotifications/RedactImportNotificationsCommand.cs @@ -0,0 +1,53 @@ +using System.IO.Compression; +using Amazon.Runtime.Internal.Util; +using Btms.Backend.Cli.Features.DownloadScenarioData; +using Btms.Business.Commands; +using Btms.Model.Ipaffs; +using Btms.SensitiveData; +using CommandLine; +using MediatR; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Refit; +using ImportNotification = Btms.Types.Ipaffs.ImportNotification; + +namespace Btms.Backend.Cli.Features.RedactImportNotifications; + +[Verb("redact-import-notifications", isDefault: false, HelpText = "Redacts Import Notification files.")] +internal class RedactImportNotificationsCommand : IRequest +{ + [Option('r', "rootFolder", Required = true, HelpText = "The root folder to search within")] + public required string RootFolder { get; set; } + + public class Handler(ILogger logger) : IRequestHandler + { + public async Task Handle(RedactImportNotificationsCommand request, CancellationToken cancellationToken) + { + DirectoryInfo di = + new DirectoryInfo(request.RootFolder); + + var files = di.GetFiles("*.json", SearchOption.AllDirectories); + + logger.LogInformation("Found {Count} files", files.Length); + + await Parallel.ForEachAsync(files, cancellationToken, async (fileInfo, ct) => + { + logger.LogInformation("Starting file {File}", fileInfo.FullName); + var json = await File.ReadAllTextAsync(fileInfo.FullName, ct); + + var options = new SensitiveDataOptions { Include = false }; + var serializer = + new SensitiveDataSerializer(Options.Create(options), NullLogger.Instance); + + var result = serializer.RedactRawJson(json, typeof(ImportNotification)); + await File.WriteAllTextAsync(fileInfo.FullName, result, ct); + + logger.LogInformation("Completed file {File}", fileInfo.FullName); + }); + + } + } + + +} \ No newline at end of file diff --git a/Btms.SensitiveData.Tests/SensitiveDataSerializerTests.cs b/Btms.SensitiveData.Tests/SensitiveDataSerializerTests.cs index 6d085df9..c88cf41b 100644 --- a/Btms.SensitiveData.Tests/SensitiveDataSerializerTests.cs +++ b/Btms.SensitiveData.Tests/SensitiveDataSerializerTests.cs @@ -20,7 +20,8 @@ public void WhenDoNotIncludeSensitiveData_ThenDataShouldBeRedacted() SimpleStringOne = "Test String One", SimpleStringTwo = "Test String Two", SimpleStringArrayOne = ["Test String Array One Item One", "Test String Array One Item Two"], - SimpleStringArrayTwo = ["Test String Array Two Item One", "Test String Array Two Item Two"] + SimpleStringArrayTwo = ["Test String Array Two Item One", "Test String Array Two Item Two"], + SimpleObjectArray = [new SimpleInnerClass() { SimpleStringOne = "Test Inner String" }] }; var json = JsonSerializer.Serialize(simpleClass); @@ -35,6 +36,7 @@ public void WhenDoNotIncludeSensitiveData_ThenDataShouldBeRedacted() result.SimpleStringArrayOne[1].Should().Be("TestRedacted"); result.SimpleStringArrayTwo[0].Should().Be("Test String Array Two Item One"); result.SimpleStringArrayTwo[1].Should().Be("Test String Array Two Item Two"); + result.SimpleObjectArray[0].SimpleStringOne.Should().Be("TestRedacted"); } [Fact] @@ -80,7 +82,8 @@ public void WhenDoNotIncludeSensitiveData_AndRequestForRawJson_ThenDataShouldBeR SimpleStringTwo = "Test String Two", SimpleStringArrayOne = ["Test String Array One Item One", "Test String Array One Item Two"], - SimpleStringArrayTwo = ["Test String Array Two Item One", "Test String Array Two Item Two"] + SimpleStringArrayTwo = ["Test String Array Two Item One", "Test String Array Two Item Two"], + SimpleObjectArray = [new SimpleInnerClass() { SimpleStringOne = "Test Inner String" }] }; var json = JsonSerializer.Serialize(simpleClass, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }); @@ -96,5 +99,6 @@ public void WhenDoNotIncludeSensitiveData_AndRequestForRawJson_ThenDataShouldBeR resultClass?.SimpleStringArrayOne[1].Should().Be("TestRedacted"); resultClass?.SimpleStringArrayTwo[0].Should().Be("Test String Array Two Item One"); resultClass?.SimpleStringArrayTwo[1].Should().Be("Test String Array Two Item Two"); + resultClass?.SimpleObjectArray[0].SimpleStringOne.Should().Be("TestRedacted"); } } \ No newline at end of file diff --git a/Btms.SensitiveData.Tests/SimpleClass.cs b/Btms.SensitiveData.Tests/SimpleClass.cs index 2a888e90..0ce06cf7 100644 --- a/Btms.SensitiveData.Tests/SimpleClass.cs +++ b/Btms.SensitiveData.Tests/SimpleClass.cs @@ -9,4 +9,13 @@ public class SimpleClass [SensitiveData] public string[] SimpleStringArrayOne { get; set; } = null!; public string[] SimpleStringArrayTwo { get; set; } = null!; + + + public SimpleInnerClass[] SimpleObjectArray { get; set; } = null!; +} + + +public class SimpleInnerClass +{ + [SensitiveData] public string SimpleStringOne { get; set; } = null!; } \ No newline at end of file diff --git a/Btms.SensitiveData/SensitiveDataSerializer.cs b/Btms.SensitiveData/SensitiveDataSerializer.cs index 5d0a6346..2409efbe 100644 --- a/Btms.SensitiveData/SensitiveDataSerializer.cs +++ b/Btms.SensitiveData/SensitiveDataSerializer.cs @@ -1,6 +1,7 @@ using System.Text.Json; using System.Text.Json.Nodes; using System.Text.Json.Serialization; +using System.Text.RegularExpressions; using Btms.Common.Extensions; using Json.Patch; using Json.Path; @@ -62,39 +63,77 @@ public string RedactRawJson(string json, Type type) var rootNode = JsonNode.Parse(json); - foreach (var sensitiveField in sensitiveFields) + var jsonPaths = EnumeratePaths(json).ToList(); + var regex = new Regex("\\[\\d\\]", RegexOptions.Compiled, TimeSpan.FromSeconds(2)); + foreach (var path in jsonPaths) { - var jsonPath = JsonPath.Parse($"$.{sensitiveField}"); - var result = jsonPath.Evaluate(rootNode); + var pathStripped = regex.Replace(path, ""); - foreach (var match in result.Matches) + if (sensitiveFields.Contains(pathStripped)) { - JsonPatch patch; - if (match.Value is JsonArray jsonArray) - { - var redactedList = jsonArray.Select(x => - { - var redactedValue = options.Value.Getter(x?.GetValue()!); - return redactedValue; - }).ToJson(); + var jsonPath = JsonPath.Parse($"$.{path}"); + var result = jsonPath.Evaluate(rootNode); - patch = new JsonPatch(PatchOperation.Replace(JsonPointer.Parse($"{match.Location!.AsJsonPointer()}"), JsonNode.Parse(redactedList))); - } - else + foreach (var match in result.Matches) { var redactedValue = options.Value.Getter(match.Value?.GetValue()!); - patch = new JsonPatch(PatchOperation.Replace(JsonPointer.Parse(match.Location!.AsJsonPointer()), redactedValue)); + var patch = new JsonPatch(PatchOperation.Replace(JsonPointer.Parse(match.Location!.AsJsonPointer()), + redactedValue)); + + var patchResult = patch.Apply(rootNode); + if (patchResult.IsSuccess) + { + rootNode = patchResult.Result; + } } + } + } + return rootNode!.ToJsonString(new JsonSerializerOptions() { WriteIndented = true }); + } - var patchResult = patch.Apply(rootNode); - if (patchResult.IsSuccess) - { - rootNode = patchResult.Result; - } + private static IEnumerable EnumeratePaths(string json) + { + var doc = JsonDocument.Parse(json).RootElement; + var queue = new Queue<(string ParentPath, JsonElement element)>(); + queue.Enqueue(("", doc)); + while (queue.Any()) + { + var (parentPath, element) = queue.Dequeue(); + foreach (var v in QueueIterator(element, parentPath, queue)) + { + yield return v; } } + } - return rootNode!.ToJsonString(); + private static IEnumerable QueueIterator(JsonElement element, string parentPath, Queue<(string ParentPath, JsonElement element)> queue) + { + switch (element.ValueKind) + { + case JsonValueKind.Object: + parentPath = parentPath == "" + ? parentPath + : parentPath + "."; + foreach (var nextEl in element.EnumerateObject()) + { + queue.Enqueue(($"{parentPath}{nextEl.Name}", nextEl.Value)); + } + break; + case JsonValueKind.Array: + foreach (var (nextEl, i) in element.EnumerateArray().Select((jsonElement, i) => (jsonElement, i))) + { + queue.Enqueue(($"{parentPath}[{i}]", nextEl)); + } + break; + case JsonValueKind.Undefined: + case JsonValueKind.String: + case JsonValueKind.Number: + case JsonValueKind.True: + case JsonValueKind.False: + case JsonValueKind.Null: + yield return parentPath; + break; + } } } \ No newline at end of file