From 58bc4b5c12151f26bfd18cf89efb8d00a593b2a5 Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Thu, 13 Feb 2025 09:21:30 +0000 Subject: [PATCH 1/7] changed the SensitiveDataSerializer to walk the json tree rather than the sensitive fields tree --- .../SensitiveDataSerializerTests.cs | 8 +- Btms.SensitiveData.Tests/SimpleClass.cs | 11 ++- Btms.SensitiveData/SensitiveDataSerializer.cs | 77 +++++++++++++------ 3 files changed, 71 insertions(+), 25 deletions(-) diff --git a/Btms.SensitiveData.Tests/SensitiveDataSerializerTests.cs b/Btms.SensitiveData.Tests/SensitiveDataSerializerTests.cs index 6d085df9..c88cf41b 100644 --- a/Btms.SensitiveData.Tests/SensitiveDataSerializerTests.cs +++ b/Btms.SensitiveData.Tests/SensitiveDataSerializerTests.cs @@ -20,7 +20,8 @@ public void WhenDoNotIncludeSensitiveData_ThenDataShouldBeRedacted() SimpleStringOne = "Test String One", SimpleStringTwo = "Test String Two", SimpleStringArrayOne = ["Test String Array One Item One", "Test String Array One Item Two"], - SimpleStringArrayTwo = ["Test String Array Two Item One", "Test String Array Two Item Two"] + SimpleStringArrayTwo = ["Test String Array Two Item One", "Test String Array Two Item Two"], + SimpleObjectArray = [new SimpleInnerClass() { SimpleStringOne = "Test Inner String" }] }; var json = JsonSerializer.Serialize(simpleClass); @@ -35,6 +36,7 @@ public void WhenDoNotIncludeSensitiveData_ThenDataShouldBeRedacted() result.SimpleStringArrayOne[1].Should().Be("TestRedacted"); result.SimpleStringArrayTwo[0].Should().Be("Test String Array Two Item One"); result.SimpleStringArrayTwo[1].Should().Be("Test String Array Two Item Two"); + result.SimpleObjectArray[0].SimpleStringOne.Should().Be("TestRedacted"); } [Fact] @@ -80,7 +82,8 @@ public void WhenDoNotIncludeSensitiveData_AndRequestForRawJson_ThenDataShouldBeR SimpleStringTwo = "Test String Two", SimpleStringArrayOne = ["Test String Array One Item One", "Test String Array One Item Two"], - SimpleStringArrayTwo = ["Test String Array Two Item One", "Test String Array Two Item Two"] + SimpleStringArrayTwo = ["Test String Array Two Item One", "Test String Array Two Item Two"], + SimpleObjectArray = [new SimpleInnerClass() { SimpleStringOne = "Test Inner String" }] }; var json = JsonSerializer.Serialize(simpleClass, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }); @@ -96,5 +99,6 @@ public void WhenDoNotIncludeSensitiveData_AndRequestForRawJson_ThenDataShouldBeR resultClass?.SimpleStringArrayOne[1].Should().Be("TestRedacted"); resultClass?.SimpleStringArrayTwo[0].Should().Be("Test String Array Two Item One"); resultClass?.SimpleStringArrayTwo[1].Should().Be("Test String Array Two Item Two"); + resultClass?.SimpleObjectArray[0].SimpleStringOne.Should().Be("TestRedacted"); } } \ No newline at end of file diff --git a/Btms.SensitiveData.Tests/SimpleClass.cs b/Btms.SensitiveData.Tests/SimpleClass.cs index 2a888e90..a72c287d 100644 --- a/Btms.SensitiveData.Tests/SimpleClass.cs +++ b/Btms.SensitiveData.Tests/SimpleClass.cs @@ -9,4 +9,13 @@ public class SimpleClass [SensitiveData] public string[] SimpleStringArrayOne { get; set; } = null!; public string[] SimpleStringArrayTwo { get; set; } = null!; -} \ No newline at end of file + + + public SimpleInnerClass[] SimpleObjectArray { get; set; } = null!; +} + + +public class SimpleInnerClass +{ + [SensitiveData] public string SimpleStringOne { get; set; } = null!; +} diff --git a/Btms.SensitiveData/SensitiveDataSerializer.cs b/Btms.SensitiveData/SensitiveDataSerializer.cs index 5d0a6346..974cf75e 100644 --- a/Btms.SensitiveData/SensitiveDataSerializer.cs +++ b/Btms.SensitiveData/SensitiveDataSerializer.cs @@ -1,6 +1,7 @@ using System.Text.Json; using System.Text.Json.Nodes; using System.Text.Json.Serialization; +using System.Text.RegularExpressions; using Btms.Common.Extensions; using Json.Patch; using Json.Path; @@ -62,39 +63,71 @@ public string RedactRawJson(string json, Type type) var rootNode = JsonNode.Parse(json); - foreach (var sensitiveField in sensitiveFields) + var jsonPaths = EnumeratePaths(json).ToList(); + var regex = new Regex("\\[\\d\\]"); + foreach (var path in jsonPaths) { - var jsonPath = JsonPath.Parse($"$.{sensitiveField}"); - var result = jsonPath.Evaluate(rootNode); + var pathStripped = regex.Replace(path, ""); - foreach (var match in result.Matches) + if (sensitiveFields.Contains(pathStripped)) { - JsonPatch patch; - if (match.Value is JsonArray jsonArray) - { - var redactedList = jsonArray.Select(x => - { - var redactedValue = options.Value.Getter(x?.GetValue()!); - return redactedValue; - }).ToJson(); + var jsonPath = JsonPath.Parse($"$.{path}"); + var result = jsonPath.Evaluate(rootNode); - patch = new JsonPatch(PatchOperation.Replace(JsonPointer.Parse($"{match.Location!.AsJsonPointer()}"), JsonNode.Parse(redactedList))); - } - else + foreach (var match in result.Matches) { var redactedValue = options.Value.Getter(match.Value?.GetValue()!); - patch = new JsonPatch(PatchOperation.Replace(JsonPointer.Parse(match.Location!.AsJsonPointer()), redactedValue)); - } + var patch = new JsonPatch(PatchOperation.Replace(JsonPointer.Parse(match.Location!.AsJsonPointer()), + redactedValue)); - - var patchResult = patch.Apply(rootNode); - if (patchResult.IsSuccess) - { - rootNode = patchResult.Result; + var patchResult = patch.Apply(rootNode); + if (patchResult.IsSuccess) + { + rootNode = patchResult.Result; + } } } } return rootNode!.ToJsonString(); } + + IEnumerable EnumeratePaths(string json) + { + var doc = JsonDocument.Parse(json).RootElement; + var queue = new Queue<(string ParentPath, JsonElement element)>(); + queue.Enqueue(("", doc)); + while (queue.Any()) + { + var (parentPath, element) = queue.Dequeue(); + switch (element.ValueKind) + { + case JsonValueKind.Object: + parentPath = parentPath == "" + ? parentPath + : parentPath + "."; + foreach (var nextEl in element.EnumerateObject()) + { + queue.Enqueue(($"{parentPath}{nextEl.Name}", nextEl.Value)); + } + break; + case JsonValueKind.Array: + foreach (var (nextEl, i) in element.EnumerateArray().Select((jsonElement, i) => (jsonElement, i))) + { + queue.Enqueue(($"{parentPath}[{i}]", nextEl)); + } + break; + case JsonValueKind.Undefined: + case JsonValueKind.String: + case JsonValueKind.Number: + case JsonValueKind.True: + case JsonValueKind.False: + case JsonValueKind.Null: + yield return parentPath; + break; + default: + throw new ArgumentOutOfRangeException(); + } + } + } } \ No newline at end of file From 9f067ad54428fe955f371cbafb7c530d9c3928d2 Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Thu, 13 Feb 2025 09:29:59 +0000 Subject: [PATCH 2/7] added cli command to run the redacting of import notifications --- .../RedactImportNotificationsCommand.cs | 65 +++++++++++++++++++ Btms.SensitiveData/SensitiveDataSerializer.cs | 2 +- 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 Btms.Backend.Cli/Features/RedactImportNotifications/RedactImportNotificationsCommand.cs diff --git a/Btms.Backend.Cli/Features/RedactImportNotifications/RedactImportNotificationsCommand.cs b/Btms.Backend.Cli/Features/RedactImportNotifications/RedactImportNotificationsCommand.cs new file mode 100644 index 00000000..1f2ca862 --- /dev/null +++ b/Btms.Backend.Cli/Features/RedactImportNotifications/RedactImportNotificationsCommand.cs @@ -0,0 +1,65 @@ +using System.IO.Compression; +using Amazon.Runtime.Internal.Util; +using Btms.Backend.Cli.Features.DownloadScenarioData; +using Btms.Business.Commands; +using Btms.Model.Ipaffs; +using Btms.SensitiveData; +using CommandLine; +using MediatR; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Refit; +using ImportNotification = Btms.Types.Ipaffs.ImportNotification; + +namespace Btms.Backend.Cli.Features.RedactImportNotifications; + +[Verb("redact-import-notifications", isDefault: false, HelpText = "Redacts Import Notification files.")] +internal class RedactImportNotificationsCommand : IRequest +{ + [Option('r', "rootFolder", Required = true, HelpText = "The root folder to search within")] + public required string RootFolder { get; set; } + + public class Handler(ILogger logger) : IRequestHandler + { + public async Task Handle(RedactImportNotificationsCommand request, CancellationToken cancellationToken) + { + DirectoryInfo di = + new DirectoryInfo(request.RootFolder); + + var files = di.GetFiles("*.json", SearchOption.AllDirectories); + + logger.LogInformation("Found {Count} files", files.Length); + + await Parallel.ForEachAsync(files, cancellationToken, async (fileInfo, ct) => + { + logger.LogInformation("Starting file {File}", fileInfo.FullName); + var json = await File.ReadAllTextAsync(fileInfo.FullName, ct); + + var options = new SensitiveDataOptions { Include = false }; + var serializer = + new SensitiveDataSerializer(Options.Create(options), NullLogger.Instance); + + var result = serializer.RedactRawJson(json, typeof(ImportNotification)); + await File.WriteAllTextAsync(fileInfo.FullName, result, ct); + + logger.LogInformation("Completed file {File}", fileInfo.FullName); + }); + + } + + + private static IBtmsApi GetApi(string environment) + { + return environment switch + { + "Local" => RestService.For("http://localhost:5002"), + "Dev" => RestService.For("https://btms-backend.dev.cdp-int.defra.cloud"), + "Test" => RestService.For("https://btms-backend.test.cdp-int.defra.cloud"), + _ => throw new ArgumentException("Invalid Environment", nameof(environment)) + }; + } + } + + +} \ No newline at end of file diff --git a/Btms.SensitiveData/SensitiveDataSerializer.cs b/Btms.SensitiveData/SensitiveDataSerializer.cs index 974cf75e..33178d43 100644 --- a/Btms.SensitiveData/SensitiveDataSerializer.cs +++ b/Btms.SensitiveData/SensitiveDataSerializer.cs @@ -89,7 +89,7 @@ public string RedactRawJson(string json, Type type) } } - return rootNode!.ToJsonString(); + return rootNode!.ToJsonString(new JsonSerializerOptions() { WriteIndented = true}); } IEnumerable EnumeratePaths(string json) From a0a6cf55b2f5ab1f2a23f6a5d9e304e3bd0d5228 Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Thu, 13 Feb 2025 09:36:16 +0000 Subject: [PATCH 3/7] fixed formatting --- Btms.SensitiveData.Tests/SimpleClass.cs | 2 +- Btms.SensitiveData/SensitiveDataSerializer.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Btms.SensitiveData.Tests/SimpleClass.cs b/Btms.SensitiveData.Tests/SimpleClass.cs index a72c287d..0ce06cf7 100644 --- a/Btms.SensitiveData.Tests/SimpleClass.cs +++ b/Btms.SensitiveData.Tests/SimpleClass.cs @@ -18,4 +18,4 @@ public class SimpleClass public class SimpleInnerClass { [SensitiveData] public string SimpleStringOne { get; set; } = null!; -} +} \ No newline at end of file diff --git a/Btms.SensitiveData/SensitiveDataSerializer.cs b/Btms.SensitiveData/SensitiveDataSerializer.cs index 33178d43..73924a14 100644 --- a/Btms.SensitiveData/SensitiveDataSerializer.cs +++ b/Btms.SensitiveData/SensitiveDataSerializer.cs @@ -89,7 +89,7 @@ public string RedactRawJson(string json, Type type) } } - return rootNode!.ToJsonString(new JsonSerializerOptions() { WriteIndented = true}); + return rootNode!.ToJsonString(new JsonSerializerOptions() { WriteIndented = true }); } IEnumerable EnumeratePaths(string json) From 93229a19f680e975ce98e0da3f3e0e3aaa7a26ef Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Thu, 13 Feb 2025 09:44:23 +0000 Subject: [PATCH 4/7] sonar issues --- Btms.SensitiveData/SensitiveDataSerializer.cs | 62 ++++++++++--------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/Btms.SensitiveData/SensitiveDataSerializer.cs b/Btms.SensitiveData/SensitiveDataSerializer.cs index 73924a14..8ec72122 100644 --- a/Btms.SensitiveData/SensitiveDataSerializer.cs +++ b/Btms.SensitiveData/SensitiveDataSerializer.cs @@ -64,7 +64,7 @@ public string RedactRawJson(string json, Type type) var rootNode = JsonNode.Parse(json); var jsonPaths = EnumeratePaths(json).ToList(); - var regex = new Regex("\\[\\d\\]"); + var regex = new Regex("\\[\\d\\]", RegexOptions.Compiled, TimeSpan.FromSeconds(2)); foreach (var path in jsonPaths) { var pathStripped = regex.Replace(path, ""); @@ -92,7 +92,7 @@ public string RedactRawJson(string json, Type type) return rootNode!.ToJsonString(new JsonSerializerOptions() { WriteIndented = true }); } - IEnumerable EnumeratePaths(string json) + private static IEnumerable EnumeratePaths(string json) { var doc = JsonDocument.Parse(json).RootElement; var queue = new Queue<(string ParentPath, JsonElement element)>(); @@ -100,34 +100,36 @@ IEnumerable EnumeratePaths(string json) while (queue.Any()) { var (parentPath, element) = queue.Dequeue(); - switch (element.ValueKind) - { - case JsonValueKind.Object: - parentPath = parentPath == "" - ? parentPath - : parentPath + "."; - foreach (var nextEl in element.EnumerateObject()) - { - queue.Enqueue(($"{parentPath}{nextEl.Name}", nextEl.Value)); - } - break; - case JsonValueKind.Array: - foreach (var (nextEl, i) in element.EnumerateArray().Select((jsonElement, i) => (jsonElement, i))) - { - queue.Enqueue(($"{parentPath}[{i}]", nextEl)); - } - break; - case JsonValueKind.Undefined: - case JsonValueKind.String: - case JsonValueKind.Number: - case JsonValueKind.True: - case JsonValueKind.False: - case JsonValueKind.Null: - yield return parentPath; - break; - default: - throw new ArgumentOutOfRangeException(); - } + yield return QueueIterator(element, parentPath, queue); + } + } + + private static string QueueIterator(JsonElement element, string parentPath, Queue<(string ParentPath, JsonElement element)> queue) + { + switch (element.ValueKind) + { + case JsonValueKind.Object: + parentPath = parentPath == "" + ? parentPath + : parentPath + "."; + foreach (var nextEl in element.EnumerateObject()) + { + queue.Enqueue(($"{parentPath}{nextEl.Name}", nextEl.Value)); + } + break; + case JsonValueKind.Array: + foreach (var (nextEl, i) in element.EnumerateArray().Select((jsonElement, i) => (jsonElement, i))) + { + queue.Enqueue(($"{parentPath}[{i}]", nextEl)); + } + break; + case JsonValueKind.Undefined: + case JsonValueKind.String: + case JsonValueKind.Number: + case JsonValueKind.True: + case JsonValueKind.False: + case JsonValueKind.Null: + return parentPath; } } } \ No newline at end of file From e6776a58c1323ca6c1321d429be9a46ecd2c8de3 Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Thu, 13 Feb 2025 09:48:36 +0000 Subject: [PATCH 5/7] removed unused code --- .../RedactImportNotificationsCommand.cs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/Btms.Backend.Cli/Features/RedactImportNotifications/RedactImportNotificationsCommand.cs b/Btms.Backend.Cli/Features/RedactImportNotifications/RedactImportNotificationsCommand.cs index 1f2ca862..fa290e2c 100644 --- a/Btms.Backend.Cli/Features/RedactImportNotifications/RedactImportNotificationsCommand.cs +++ b/Btms.Backend.Cli/Features/RedactImportNotifications/RedactImportNotificationsCommand.cs @@ -47,18 +47,6 @@ await Parallel.ForEachAsync(files, cancellationToken, async (fileInfo, ct) => }); } - - - private static IBtmsApi GetApi(string environment) - { - return environment switch - { - "Local" => RestService.For("http://localhost:5002"), - "Dev" => RestService.For("https://btms-backend.dev.cdp-int.defra.cloud"), - "Test" => RestService.For("https://btms-backend.test.cdp-int.defra.cloud"), - _ => throw new ArgumentException("Invalid Environment", nameof(environment)) - }; - } } From 1762a5570b2e8bcb901777ac5a40005a92286948 Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Thu, 13 Feb 2025 09:57:06 +0000 Subject: [PATCH 6/7] fixed formatting --- Btms.SensitiveData/SensitiveDataSerializer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Btms.SensitiveData/SensitiveDataSerializer.cs b/Btms.SensitiveData/SensitiveDataSerializer.cs index 8ec72122..e1486a84 100644 --- a/Btms.SensitiveData/SensitiveDataSerializer.cs +++ b/Btms.SensitiveData/SensitiveDataSerializer.cs @@ -92,7 +92,7 @@ public string RedactRawJson(string json, Type type) return rootNode!.ToJsonString(new JsonSerializerOptions() { WriteIndented = true }); } - private static IEnumerable EnumeratePaths(string json) + private static IEnumerable EnumeratePaths(string json) { var doc = JsonDocument.Parse(json).RootElement; var queue = new Queue<(string ParentPath, JsonElement element)>(); From fddb8644fe57297d19613cee803f893c3f4846fd Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Thu, 13 Feb 2025 10:06:26 +0000 Subject: [PATCH 7/7] fixed sonar issues --- Btms.SensitiveData/SensitiveDataSerializer.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Btms.SensitiveData/SensitiveDataSerializer.cs b/Btms.SensitiveData/SensitiveDataSerializer.cs index e1486a84..2409efbe 100644 --- a/Btms.SensitiveData/SensitiveDataSerializer.cs +++ b/Btms.SensitiveData/SensitiveDataSerializer.cs @@ -100,11 +100,14 @@ private static IEnumerable EnumeratePaths(string json) while (queue.Any()) { var (parentPath, element) = queue.Dequeue(); - yield return QueueIterator(element, parentPath, queue); + foreach (var v in QueueIterator(element, parentPath, queue)) + { + yield return v; + } } } - private static string QueueIterator(JsonElement element, string parentPath, Queue<(string ParentPath, JsonElement element)> queue) + private static IEnumerable QueueIterator(JsonElement element, string parentPath, Queue<(string ParentPath, JsonElement element)> queue) { switch (element.ValueKind) { @@ -129,7 +132,8 @@ private static string QueueIterator(JsonElement element, string parentPath, Queu case JsonValueKind.True: case JsonValueKind.False: case JsonValueKind.Null: - return parentPath; + yield return parentPath; + break; } } } \ No newline at end of file