From 25c7c782e0959b1c7b59bcfe8c6c4e241675925c Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Tue, 11 Feb 2025 07:41:12 +0000 Subject: [PATCH 01/13] Added unique constraint on decisionNumber --- Btms.Backend.Data/Mongo/MongoIndexService.cs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Btms.Backend.Data/Mongo/MongoIndexService.cs b/Btms.Backend.Data/Mongo/MongoIndexService.cs index 652e6ac7..ac203f02 100644 --- a/Btms.Backend.Data/Mongo/MongoIndexService.cs +++ b/Btms.Backend.Data/Mongo/MongoIndexService.cs @@ -12,18 +12,20 @@ public Task StartAsync(CancellationToken cancellationToken) { return Task.WhenAll( CreateIndex("MatchReferenceIdx", - Builders.IndexKeys.Ascending(n => n._MatchReference), cancellationToken), + Builders.IndexKeys.Ascending(n => n._MatchReference), cancellationToken: cancellationToken), CreateIndex("Created", - Builders.IndexKeys.Ascending(n => n.Created), cancellationToken), + Builders.IndexKeys.Ascending(n => n.Created), cancellationToken: cancellationToken), CreateIndex("CreatedSource", - Builders.IndexKeys.Ascending(n => n.CreatedSource), cancellationToken), + Builders.IndexKeys.Ascending(n => n.CreatedSource), cancellationToken: cancellationToken), CreateIndex("MatchReferenceIdx", - Builders.IndexKeys.Ascending(m => m._MatchReferences), cancellationToken), + Builders.IndexKeys.Ascending(m => m._MatchReferences), cancellationToken: cancellationToken), CreateIndex("Created", - Builders.IndexKeys.Ascending(m => m.Created), cancellationToken), + Builders.IndexKeys.Ascending(m => m.Created), cancellationToken: cancellationToken), CreateIndex("CreatedSource", - Builders.IndexKeys.Ascending(m => m.CreatedSource), cancellationToken) + Builders.IndexKeys.Ascending(m => m.CreatedSource), cancellationToken: cancellationToken), + CreateIndex("UniqueDecisionNumber", + Builders.IndexKeys.Ascending(new StringFieldDefinition("decisions.header.decisionNumber")), true, cancellationToken) ); @@ -34,7 +36,7 @@ public Task StopAsync(CancellationToken cancellationToken) return Task.CompletedTask; } - private async Task CreateIndex(string name, IndexKeysDefinition keys, CancellationToken cancellationToken) + private async Task CreateIndex(string name, IndexKeysDefinition keys, bool unique = false, CancellationToken cancellationToken = default) { try { @@ -43,6 +45,7 @@ private async Task CreateIndex(string name, IndexKeysDefinition keys, Canc { Name = name, Background = true, + Unique = unique, }); await database.GetCollection(typeof(T).Name).Indexes.CreateOneAsync(indexModel, cancellationToken: cancellationToken); } From 5c3b803350105959ebbbb6013ddc3374ce78e722 Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Tue, 11 Feb 2025 07:41:12 +0000 Subject: [PATCH 02/13] Added unique constraint on decisionNumber --- Btms.Backend.Data/Mongo/MongoIndexService.cs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Btms.Backend.Data/Mongo/MongoIndexService.cs b/Btms.Backend.Data/Mongo/MongoIndexService.cs index 05fa10dc..8f56b640 100644 --- a/Btms.Backend.Data/Mongo/MongoIndexService.cs +++ b/Btms.Backend.Data/Mongo/MongoIndexService.cs @@ -12,18 +12,20 @@ public Task StartAsync(CancellationToken cancellationToken) { return Task.WhenAll( CreateIndex("MatchReferenceIdx", - Builders.IndexKeys.Ascending(n => n._MatchReference), cancellationToken), + Builders.IndexKeys.Ascending(n => n._MatchReference), cancellationToken: cancellationToken), CreateIndex("Created", - Builders.IndexKeys.Ascending(n => n.Created), cancellationToken), + Builders.IndexKeys.Ascending(n => n.Created), cancellationToken: cancellationToken), CreateIndex("CreatedSource", - Builders.IndexKeys.Ascending(n => n.CreatedSource), cancellationToken), + Builders.IndexKeys.Ascending(n => n.CreatedSource), cancellationToken: cancellationToken), CreateIndex("MatchReferenceIdx", - Builders.IndexKeys.Ascending(m => m._MatchReferences), cancellationToken), + Builders.IndexKeys.Ascending(m => m._MatchReferences), cancellationToken: cancellationToken), CreateIndex("Created", - Builders.IndexKeys.Ascending(m => m.Created), cancellationToken), + Builders.IndexKeys.Ascending(m => m.Created), cancellationToken: cancellationToken), CreateIndex("CreatedSource", - Builders.IndexKeys.Ascending(m => m.CreatedSource), cancellationToken) + Builders.IndexKeys.Ascending(m => m.CreatedSource), cancellationToken: cancellationToken), + CreateIndex("UniqueDecisionNumber", + Builders.IndexKeys.Ascending(new StringFieldDefinition("decisions.header.decisionNumber")), true, cancellationToken) ); @@ -34,7 +36,7 @@ public Task StopAsync(CancellationToken cancellationToken) return Task.CompletedTask; } - private async Task CreateIndex(string name, IndexKeysDefinition keys, CancellationToken cancellationToken) + private async Task CreateIndex(string name, IndexKeysDefinition keys, bool unique = false, CancellationToken cancellationToken = default) { try { @@ -43,6 +45,7 @@ private async Task CreateIndex(string name, IndexKeysDefinition keys, Canc { Name = name, Background = true, + Unique = unique, }); await database.GetCollection(typeof(T).Name).Indexes.CreateOneAsync(indexModel, cancellationToken: cancellationToken); } From 9e2fb789e422f2707373489682c56c8bfc816f3c Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Tue, 11 Feb 2025 16:39:48 +0000 Subject: [PATCH 03/13] Change the decision saving to be wrapped within the consumer transaction to avoid duplicate decision numbers --- Btms.Backend.Data/Mongo/MongoIndexService.cs | 17 +++++----- .../Decisions/DecisionMessageBuilderTests.cs | 3 +- .../Decisions/DecisionServiceTests.cs | 11 ++++++- .../Decisions/NoMatchDecisionsTest.cs | 15 ++++++--- .../Services/Decisions/DecisionContext.cs | 3 ++ .../Services/Decisions/DecisionService.cs | 31 ++++++++++++++++++- .../AlvsClearanceRequestConsumer.cs | 5 +-- Btms.Consumers/NotificationConsumer.cs | 10 +++++- 8 files changed, 75 insertions(+), 20 deletions(-) diff --git a/Btms.Backend.Data/Mongo/MongoIndexService.cs b/Btms.Backend.Data/Mongo/MongoIndexService.cs index 8f56b640..05fa10dc 100644 --- a/Btms.Backend.Data/Mongo/MongoIndexService.cs +++ b/Btms.Backend.Data/Mongo/MongoIndexService.cs @@ -12,20 +12,18 @@ public Task StartAsync(CancellationToken cancellationToken) { return Task.WhenAll( CreateIndex("MatchReferenceIdx", - Builders.IndexKeys.Ascending(n => n._MatchReference), cancellationToken: cancellationToken), + Builders.IndexKeys.Ascending(n => n._MatchReference), cancellationToken), CreateIndex("Created", - Builders.IndexKeys.Ascending(n => n.Created), cancellationToken: cancellationToken), + Builders.IndexKeys.Ascending(n => n.Created), cancellationToken), CreateIndex("CreatedSource", - Builders.IndexKeys.Ascending(n => n.CreatedSource), cancellationToken: cancellationToken), + Builders.IndexKeys.Ascending(n => n.CreatedSource), cancellationToken), CreateIndex("MatchReferenceIdx", - Builders.IndexKeys.Ascending(m => m._MatchReferences), cancellationToken: cancellationToken), + Builders.IndexKeys.Ascending(m => m._MatchReferences), cancellationToken), CreateIndex("Created", - Builders.IndexKeys.Ascending(m => m.Created), cancellationToken: cancellationToken), + Builders.IndexKeys.Ascending(m => m.Created), cancellationToken), CreateIndex("CreatedSource", - Builders.IndexKeys.Ascending(m => m.CreatedSource), cancellationToken: cancellationToken), - CreateIndex("UniqueDecisionNumber", - Builders.IndexKeys.Ascending(new StringFieldDefinition("decisions.header.decisionNumber")), true, cancellationToken) + Builders.IndexKeys.Ascending(m => m.CreatedSource), cancellationToken) ); @@ -36,7 +34,7 @@ public Task StopAsync(CancellationToken cancellationToken) return Task.CompletedTask; } - private async Task CreateIndex(string name, IndexKeysDefinition keys, bool unique = false, CancellationToken cancellationToken = default) + private async Task CreateIndex(string name, IndexKeysDefinition keys, CancellationToken cancellationToken) { try { @@ -45,7 +43,6 @@ private async Task CreateIndex(string name, IndexKeysDefinition keys, bool { Name = name, Background = true, - Unique = unique, }); await database.GetCollection(typeof(T).Name).Indexes.CreateOneAsync(indexModel, cancellationToken: cancellationToken); } diff --git a/Btms.Business.Tests/Services/Decisions/DecisionMessageBuilderTests.cs b/Btms.Business.Tests/Services/Decisions/DecisionMessageBuilderTests.cs index 3eb81e2f..37e42532 100644 --- a/Btms.Business.Tests/Services/Decisions/DecisionMessageBuilderTests.cs +++ b/Btms.Business.Tests/Services/Decisions/DecisionMessageBuilderTests.cs @@ -100,7 +100,8 @@ private static DecisionContext CreateDecisionContext() ] } ], - new MatchingResult() + new MatchingResult(), + "TestMessageId" ); } diff --git a/Btms.Business.Tests/Services/Decisions/DecisionServiceTests.cs b/Btms.Business.Tests/Services/Decisions/DecisionServiceTests.cs index 7dd2c42e..7f0fbd13 100644 --- a/Btms.Business.Tests/Services/Decisions/DecisionServiceTests.cs +++ b/Btms.Business.Tests/Services/Decisions/DecisionServiceTests.cs @@ -1,3 +1,6 @@ +using Btms.Backend.Data; +using Btms.Backend.Data.InMemory; +using Btms.Business.Builders; using Btms.Business.Services.Decisions; using Btms.Business.Services.Decisions.Finders; using Btms.Business.Services.Matching; @@ -96,6 +99,10 @@ public DecisionServiceTests() _serviceCollection.AddSingleton(); _serviceCollection.AddSingleton(Substitute.For>()); _serviceCollection.AddSingleton(Substitute.For()); + _serviceCollection.AddSingleton(); + _serviceCollection.AddSingleton(); + _serviceCollection.AddLogging(); + _serviceCollection.AddSingleton(); } private ServiceProvider ConfigureDecisionFinders(ImportNotification notification, string[] checkCodes) @@ -146,6 +153,7 @@ private static DecisionContext CreateDecisionContext(ImportNotificationTypeEnum? [ new Movement { + EntryReference = "movement-1", Id = "movement-1", BtmsStatus = MovementStatus.Default(), Items = @@ -158,7 +166,8 @@ private static DecisionContext CreateDecisionContext(ImportNotificationTypeEnum? ] } ], - matchingResult + matchingResult, + "TestMessageId" ); } } \ No newline at end of file diff --git a/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs b/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs index 33db4f5e..d94b894e 100644 --- a/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs +++ b/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs @@ -1,3 +1,4 @@ +using Btms.Backend.Data.InMemory; using Btms.Business.Builders; using Btms.Business.Services.Decisions; using Btms.Business.Services.Decisions.Finders; @@ -26,13 +27,16 @@ public async Task WhenClearanceRequest_HasNotMatch_AndNoChecks_ThenNoDecisionSho // Arrange var movements = GenerateMovements(false); - var sut = new DecisionService(NullLogger.Instance, Array.Empty()); + var sut = new DecisionService(NullLogger.Instance, + Array.Empty(), + new MovementBuilderFactory(new DecisionStatusFinder(), NullLogger.Instance), + new MemoryMongoDbContext()); var matchingResult = new MatchingResult(); matchingResult.AddDocumentNoMatch(movements[0].Id!, movements[0].Items[0].ItemNumber!.Value, movements[0].Items[0].Documents?[0].DocumentReference!); // Act - var decisionResult = await sut.Process(new DecisionContext(new List(), movements, matchingResult, true), CancellationToken.None); + var decisionResult = await sut.Process(new DecisionContext(new List(), movements, matchingResult, "TestMessageId",true), CancellationToken.None); // Assert decisionResult.Should().NotBeNull(); @@ -48,13 +52,16 @@ public async Task WhenClearanceRequest_HasNotMatch_ThenDecisionCodeShouldBeNoMat var movements = GenerateMovements(true); movements[0].Items[0].Checks = [new Check() { CheckCode = "TEST" }]; - var sut = new DecisionService(NullLogger.Instance, Array.Empty()); + var sut = new DecisionService(NullLogger.Instance, + Array.Empty(), + new MovementBuilderFactory(new DecisionStatusFinder(), NullLogger.Instance), + new MemoryMongoDbContext()); var matchingResult = new MatchingResult(); matchingResult.AddDocumentNoMatch(movements[0].Id!, movements[0].Items[0].ItemNumber!.Value, movements[0].Items[0].Documents?[0].DocumentReference!); // Act - var decisionResult = await sut.Process(new DecisionContext(new List(), movements, matchingResult, true), CancellationToken.None); + var decisionResult = await sut.Process(new DecisionContext(new List(), movements, matchingResult, "TestMessageId", true), CancellationToken.None); // Assert decisionResult.Should().NotBeNull(); diff --git a/Btms.Business/Services/Decisions/DecisionContext.cs b/Btms.Business/Services/Decisions/DecisionContext.cs index e93690aa..4a929eda 100644 --- a/Btms.Business/Services/Decisions/DecisionContext.cs +++ b/Btms.Business/Services/Decisions/DecisionContext.cs @@ -8,6 +8,7 @@ public class DecisionContext( List notifications, List movements, MatchingResult matchingResult, + string messageId, bool generateNoMatch = false) { public List Notifications { get; } = notifications; @@ -15,4 +16,6 @@ public class DecisionContext( public MatchingResult MatchingResult { get; } = matchingResult; public bool GenerateNoMatch { get; } = generateNoMatch; + + public string MessageId { get; } = messageId; } \ No newline at end of file diff --git a/Btms.Business/Services/Decisions/DecisionService.cs b/Btms.Business/Services/Decisions/DecisionService.cs index f5cd124a..3310b378 100644 --- a/Btms.Business/Services/Decisions/DecisionService.cs +++ b/Btms.Business/Services/Decisions/DecisionService.cs @@ -1,11 +1,16 @@ using System.Diagnostics.CodeAnalysis; +using Btms.Backend.Data; +using Btms.Business.Builders; using Btms.Business.Services.Decisions.Finders; +using Btms.Model.Cds; using Btms.Model.Ipaffs; +using Btms.Types.Alvs.Mapping; using Microsoft.Extensions.Logging; namespace Btms.Business.Services.Decisions; -public class DecisionService(ILogger logger, IEnumerable decisionFinders) : IDecisionService +public class DecisionService(ILogger logger, IEnumerable decisionFinders, + MovementBuilderFactory movementBuilderFactory, IMongoDbContext dbContext) : IDecisionService { public async Task Process(DecisionContext decisionContext, CancellationToken cancellationToken) { @@ -18,6 +23,30 @@ public async Task Process(DecisionContext decisionContext, Cance decisionResult.AddDecisionMessage(message); } + var notificationContext = decisionContext.Notifications + .Select(n => new DecisionImportNotifications + { + Id = n.Id!, + Version = n.Version, + Created = n.Created, + Updated = n.Updated, + UpdatedEntity = n.UpdatedEntity, + CreatedSource = n.CreatedSource!.Value, + UpdatedSource = n.UpdatedSource!.Value + }) + .ToList(); + + foreach (var decisionResultDecisionsMessage in decisionResult.DecisionsMessages) + { + var internalDecision = DecisionMapper.Map(decisionResultDecisionsMessage); + var m = decisionContext.Movements.First(m => m.Id!.Equals(decisionResultDecisionsMessage.Header?.EntryReference)); + var existingMovementBuilder = movementBuilderFactory + .From(m) + .MergeDecision(decisionContext.MessageId, internalDecision, notificationContext); + m = existingMovementBuilder.Build(); + await dbContext.Movements.Update(m, cancellationToken); + } + return decisionResult; } diff --git a/Btms.Consumers/AlvsClearanceRequestConsumer.cs b/Btms.Consumers/AlvsClearanceRequestConsumer.cs index 51c4524a..13696d18 100644 --- a/Btms.Consumers/AlvsClearanceRequestConsumer.cs +++ b/Btms.Consumers/AlvsClearanceRequestConsumer.cs @@ -71,14 +71,15 @@ public async Task OnHandle(AlvsClearanceRequest message, CancellationToken cance new MatchingContext(linkResult.Notifications, linkResult.Movements), Context.CancellationToken); var decisionContext = - new DecisionContext(linkResult.Notifications, linkResult.Movements, matchResult, true); + new DecisionContext(linkResult.Notifications, linkResult.Movements, matchResult, messageId, true); var decisionResult = await decisionService.Process(decisionContext, Context.CancellationToken); await validationService.PostDecision(linkResult, decisionResult, Context.CancellationToken); await dbContext.SaveChangesAsync(Context.CancellationToken); - await Context.Bus.PublishDecisions(messageId, decisionResult, decisionContext, cancellationToken: cancellationToken); + //Since we are doing this in the Decision Service at the moment, and there are no consumers of the BTMS message do we need this here + ////await Context.Bus.PublishDecisions(messageId, decisionResult, decisionContext, cancellationToken: cancellationToken); } else { diff --git a/Btms.Consumers/NotificationConsumer.cs b/Btms.Consumers/NotificationConsumer.cs index d8d73ac8..a928e70b 100644 --- a/Btms.Consumers/NotificationConsumer.cs +++ b/Btms.Consumers/NotificationConsumer.cs @@ -11,6 +11,8 @@ using Btms.Business.Services.Validating; using Btms.Model.Cds; using DecisionContext = Btms.Business.Services.Decisions.DecisionContext; +using Btms.Business.Builders; +using Btms.Types.Alvs.Mapping; namespace Btms.Consumers; @@ -75,13 +77,19 @@ public async Task OnHandle(ImportNotification message, CancellationToken cancell var matchResult = await matchingService.Process( new MatchingContext(notifications, linkResult.Movements), Context.CancellationToken); - var decisionContext = new DecisionContext(notifications, linkResult.Movements, matchResult); + var decisionContext = new DecisionContext(notifications, linkResult.Movements, matchResult, messageId); var decisionResult = await decisionService.Process(decisionContext, Context.CancellationToken); await validationService.PostDecision(linkResult, decisionResult, Context.CancellationToken); + + + + await dbContext.SaveChangesAsync(Context.CancellationToken); + + await Context.Bus.PublishDecisions(messageId, decisionResult, decisionContext, cancellationToken: cancellationToken); } else if (preProcessingResult.IsDeleted()) From 72018709aaf7dbc16aecae3c6ba236f0c3b1f572 Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Tue, 11 Feb 2025 16:54:07 +0000 Subject: [PATCH 04/13] fixed merge conflict --- .../Services/Decisions/DecisionService.cs | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/Btms.Business/Services/Decisions/DecisionService.cs b/Btms.Business/Services/Decisions/DecisionService.cs index 3310b378..6a02128e 100644 --- a/Btms.Business/Services/Decisions/DecisionService.cs +++ b/Btms.Business/Services/Decisions/DecisionService.cs @@ -2,6 +2,7 @@ using Btms.Backend.Data; using Btms.Business.Builders; using Btms.Business.Services.Decisions.Finders; +using Btms.Business.Services.Matching; using Btms.Model.Cds; using Btms.Model.Ipaffs; using Btms.Types.Alvs.Mapping; @@ -60,7 +61,10 @@ private Task DeriveDecision(DecisionContext decisionContext) { if (decisionContext.HasChecks(noMatch.MovementId, noMatch.ItemNumber)) { - decisionsResult.AddDecision(noMatch.MovementId, noMatch.ItemNumber, noMatch.DocumentReference, null, DecisionCode.X00); + var movement = decisionContext.Movements.First(x => x.Id == noMatch.MovementId); + var checkCodes = movement.Items.First(x => x.ItemNumber == noMatch.ItemNumber).Checks?.Select(x => x.CheckCode).Where(x => x != null).Cast().ToArray(); + + HandleNoMatch(checkCodes, decisionsResult, noMatch); } } } @@ -80,6 +84,31 @@ private Task DeriveDecision(DecisionContext decisionContext) return Task.FromResult(decisionsResult); } + private static void HandleNoMatch(string[]? checkCodes, DecisionResult decisionsResult, DocumentNoMatch noMatch) + { + if (checkCodes != null) + { + foreach (var checkCode in checkCodes) + { + string? reason = null; + + if (checkCode is "H220") + { + reason = + "A Customs Declaration with a GMS product has been selected for HMI inspection. In IPAFFS create a CHEDPP and amend your licence to reference it. If a CHEDPP exists, amend your licence to reference it. Failure to do so will delay your Customs release"; + } + + decisionsResult.AddDecision(noMatch.MovementId, noMatch.ItemNumber, + noMatch.DocumentReference, checkCode, DecisionCode.X00, reason); + } + } + else + { + decisionsResult.AddDecision(noMatch.MovementId, noMatch.ItemNumber, + noMatch.DocumentReference, null, DecisionCode.X00); + } + } + private DecisionFinderResult[] GetDecisions(ImportNotification notification, string[]? checkCodes) { var results = new List(); From 1263fc589def134198b61092beef2c3b03f6182d Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Tue, 11 Feb 2025 16:55:03 +0000 Subject: [PATCH 05/13] updated NotificationConsumer --- Btms.Consumers/NotificationConsumer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Btms.Consumers/NotificationConsumer.cs b/Btms.Consumers/NotificationConsumer.cs index a928e70b..c2516a10 100644 --- a/Btms.Consumers/NotificationConsumer.cs +++ b/Btms.Consumers/NotificationConsumer.cs @@ -90,7 +90,7 @@ public async Task OnHandle(ImportNotification message, CancellationToken cancell - await Context.Bus.PublishDecisions(messageId, decisionResult, decisionContext, cancellationToken: cancellationToken); + ////await Context.Bus.PublishDecisions(messageId, decisionResult, decisionContext, cancellationToken: cancellationToken); } else if (preProcessingResult.IsDeleted()) { From e90275039dc8e5a8ef3b63ff94e18a05b8200b23 Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Tue, 11 Feb 2025 17:06:51 +0000 Subject: [PATCH 06/13] fixed formatting --- .../Services/Decisions/NoMatchDecisionsTest.cs | 2 +- Btms.Consumers/NotificationConsumer.cs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs b/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs index a696619b..34d3f4ca 100644 --- a/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs +++ b/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs @@ -60,7 +60,7 @@ public async Task WhenClearanceRequest_HasNotMatch_AndNoChecks_ThenNoDecisionSho matchingResult.AddDocumentNoMatch(movements[0].Id!, movements[0].Items[0].ItemNumber!.Value, movements[0].Items[0].Documents?[0].DocumentReference!); // Act - var decisionResult = await sut.Process(new DecisionContext(new List(), movements, matchingResult, "TestMessageId",true), CancellationToken.None); + var decisionResult = await sut.Process(new DecisionContext(new List(), movements, matchingResult, "TestMessageId", true), CancellationToken.None); // Assert decisionResult.Should().NotBeNull(); diff --git a/Btms.Consumers/NotificationConsumer.cs b/Btms.Consumers/NotificationConsumer.cs index c2516a10..93b8a05e 100644 --- a/Btms.Consumers/NotificationConsumer.cs +++ b/Btms.Consumers/NotificationConsumer.cs @@ -82,13 +82,13 @@ public async Task OnHandle(ImportNotification message, CancellationToken cancell await validationService.PostDecision(linkResult, decisionResult, Context.CancellationToken); - - + + await dbContext.SaveChangesAsync(Context.CancellationToken); - + ////await Context.Bus.PublishDecisions(messageId, decisionResult, decisionContext, cancellationToken: cancellationToken); } From 3c3f12797db71cc70b65f487f03608a9542897e5 Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Tue, 11 Feb 2025 17:20:58 +0000 Subject: [PATCH 07/13] fixed issue --- .../Services/Decisions/NoMatchDecisionsTest.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs b/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs index 34d3f4ca..7a7eeb8c 100644 --- a/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs +++ b/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs @@ -27,13 +27,16 @@ public async Task WhenClearanceRequest_HasNotMatch_AndH220Checks_ThenNoDecisionS // Arrange var movement = GenerateMovementWithH220Checks(); - var sut = new DecisionService(NullLogger.Instance, Array.Empty()); + var sut = new DecisionService(NullLogger.Instance, + Array.Empty(), + new MovementBuilderFactory(new DecisionStatusFinder(), NullLogger.Instance), + new MemoryMongoDbContext()); var matchingResult = new MatchingResult(); matchingResult.AddDocumentNoMatch(movement.Id!, movement.Items[0].ItemNumber!.Value, movement.Items[0].Documents?[0].DocumentReference!); // Act - var decisionResult = await sut.Process(new DecisionContext(new List(), [movement], matchingResult, true), CancellationToken.None); + var decisionResult = await sut.Process(new DecisionContext(new List(), [movement], matchingResult,"TestMessageId", true), CancellationToken.None); // Assert decisionResult.Should().NotBeNull(); From aa78687ddf0db17646a91c37e83a083ab8d9538a Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Tue, 11 Feb 2025 17:26:25 +0000 Subject: [PATCH 08/13] formatting --- Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs b/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs index 7a7eeb8c..63a08d74 100644 --- a/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs +++ b/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs @@ -36,7 +36,7 @@ public async Task WhenClearanceRequest_HasNotMatch_AndH220Checks_ThenNoDecisionS matchingResult.AddDocumentNoMatch(movement.Id!, movement.Items[0].ItemNumber!.Value, movement.Items[0].Documents?[0].DocumentReference!); // Act - var decisionResult = await sut.Process(new DecisionContext(new List(), [movement], matchingResult,"TestMessageId", true), CancellationToken.None); + var decisionResult = await sut.Process(new DecisionContext(new List(), [movement], matchingResult, "TestMessageId", true), CancellationToken.None); // Assert decisionResult.Should().NotBeNull(); From c3f889715527855b465f592b063f176d95b43903 Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Wed, 12 Feb 2025 06:13:31 +0000 Subject: [PATCH 09/13] updated to add skipped audit entries --- Btms.Business/Builders/MovementBuilder.cs | 16 +++++++-- .../ImportNotificationPreProcessor.cs | 12 +++++-- .../PreProcessing/MovementPreProcessor.cs | 34 +++++++++---------- .../AlvsClearanceRequestConsumer.cs | 11 ++++-- Btms.Consumers/NotificationConsumer.cs | 19 ++--------- 5 files changed, 50 insertions(+), 42 deletions(-) diff --git a/Btms.Business/Builders/MovementBuilder.cs b/Btms.Business/Builders/MovementBuilder.cs index debf0e72..d99cd23b 100644 --- a/Btms.Business/Builders/MovementBuilder.cs +++ b/Btms.Business/Builders/MovementBuilder.cs @@ -1,12 +1,11 @@ -using System.Diagnostics.CodeAnalysis; using Btms.Business.Extensions; using Btms.Common.Extensions; using Btms.Model; using Btms.Model.Auditing; using Btms.Model.Cds; using Btms.Model.ChangeLog; -using Btms.Model.Ipaffs; using Microsoft.Extensions.Logging; +using System.Diagnostics.CodeAnalysis; namespace Btms.Business.Builders; @@ -198,6 +197,19 @@ public AuditEntry UpdateAuditEntry(string messageId, CreatedBySystem source, Cha return auditEntry; } + public AuditEntry SkippedAuditEntry(string messageId, CreatedBySystem source) + { + GuardNullMovement(); + + var auditEntry = AuditEntry.CreateSkippedVersion( + messageId, + _movement.ClearanceRequests[0].Header!.EntryVersionNumber.GetValueOrDefault(), + _movement.UpdatedSource, + source); + + return auditEntry; + } + private static string BuildNormalizedDecisionPath(string fullPath) { return fullPath.Replace("RAW/DECISIONS/", ""); diff --git a/Btms.Business/Pipelines/PreProcessing/ImportNotificationPreProcessor.cs b/Btms.Business/Pipelines/PreProcessing/ImportNotificationPreProcessor.cs index 1c866290..2f233787 100644 --- a/Btms.Business/Pipelines/PreProcessing/ImportNotificationPreProcessor.cs +++ b/Btms.Business/Pipelines/PreProcessing/ImportNotificationPreProcessor.cs @@ -59,14 +59,20 @@ public class ImportNotificationPreProcessor(IMongoDbContext dbContext, ILogger result; if (internalNotification.UpdatedSource.TrimMicroseconds() == existingNotification.UpdatedSource.TrimMicroseconds()) { - return PreProcessResult.AlreadyProcessed(existingNotification); + result = PreProcessResult.AlreadyProcessed(existingNotification); + } + else + { + logger.MessageSkipped(preProcessingContext.MessageId, preProcessingContext.Message.ReferenceNumber!); + result = PreProcessResult.Skipped(existingNotification); } - logger.MessageSkipped(preProcessingContext.MessageId, preProcessingContext.Message.ReferenceNumber!); - return PreProcessResult.Skipped(existingNotification); + internalNotification.Skipped(preProcessingContext.MessageId, internalNotification.Version.GetValueOrDefault()); + return result; } } \ No newline at end of file diff --git a/Btms.Business/Pipelines/PreProcessing/MovementPreProcessor.cs b/Btms.Business/Pipelines/PreProcessing/MovementPreProcessor.cs index 69d1f29a..9d79b3aa 100644 --- a/Btms.Business/Pipelines/PreProcessing/MovementPreProcessor.cs +++ b/Btms.Business/Pipelines/PreProcessing/MovementPreProcessor.cs @@ -2,7 +2,6 @@ using Btms.Business.Builders; using Btms.Model; using Btms.Model.Auditing; -using Btms.Model.ChangeLog; using Btms.Types.Alvs; using Btms.Types.Alvs.Mapping; using Microsoft.Extensions.Logging; @@ -13,15 +12,12 @@ public class MovementPreProcessor(IMongoDbContext dbContext, ILogger> Process(PreProcessingContext preProcessingContext) { - var internalClearanceRequest = AlvsClearanceRequestMapper.Map(preProcessingContext.Message); var mb = movementBuilderFactory.From(internalClearanceRequest); var existingMovement = await dbContext.Movements.Find(mb.Id); if (existingMovement is null) { - // ArgumentNullException.ThrowIfNull(movement); - var auditEntry = mb.CreateAuditEntry( preProcessingContext.MessageId, CreatedBySystem.Cds @@ -33,11 +29,10 @@ public async Task> Process(PreProcessingContext existingMovement.ClearanceRequests[0].Header?.EntryVersionNumber) + var existingBuilder = movementBuilderFactory.From(existingMovement); + if (mb.IsEntryVersionNumberGreaterThan(existingMovement.ClearanceRequests[0].Header?.EntryVersionNumber)) { - var existingBuilder = movementBuilderFactory.From(existingMovement); - // var changeSet = movement.ClearanceRequests[^1].GenerateChangeSet(existingMovement.ClearanceRequests[0]); var changeSet = mb.GenerateChangeSet(existingBuilder); var auditEntry = mb.UpdateAuditEntry( @@ -50,25 +45,30 @@ public async Task> Process(PreProcessingContext - // x.Header?.EntryReference == - // movement.ClearanceRequests[0].Header?.EntryReference); - // existingMovement.ClearanceRequests.AddRange(movement.ClearanceRequests); - // - // existingMovement.Items.AddRange(movement.Items); - await dbContext.Movements.Update(existingMovement); return PreProcessResult.Changed(existingMovement, changeSet); } + PreProcessingResult result; + if (mb.IsEntryVersionNumberEqualTo(existingMovement.ClearanceRequests[0].Header?.EntryVersionNumber)) { - return PreProcessResult.AlreadyProcessed(existingMovement); + result = PreProcessResult.AlreadyProcessed(existingMovement); } + else + { + logger.MessageSkipped(preProcessingContext.MessageId, preProcessingContext.Message.Header?.EntryReference!); + result = PreProcessResult.Skipped(existingMovement); + } + + var skippedAuditEntry = existingBuilder.SkippedAuditEntry( + preProcessingContext.MessageId, + CreatedBySystem.Cds); + + existingBuilder.Update(skippedAuditEntry); - logger.MessageSkipped(preProcessingContext.MessageId, preProcessingContext.Message.Header?.EntryReference!); - return PreProcessResult.Skipped(existingMovement); + return result; } } \ No newline at end of file diff --git a/Btms.Consumers/AlvsClearanceRequestConsumer.cs b/Btms.Consumers/AlvsClearanceRequestConsumer.cs index 13696d18..85942d67 100644 --- a/Btms.Consumers/AlvsClearanceRequestConsumer.cs +++ b/Btms.Consumers/AlvsClearanceRequestConsumer.cs @@ -76,10 +76,7 @@ public async Task OnHandle(AlvsClearanceRequest message, CancellationToken cance await validationService.PostDecision(linkResult, decisionResult, Context.CancellationToken); - await dbContext.SaveChangesAsync(Context.CancellationToken); - //Since we are doing this in the Decision Service at the moment, and there are no consumers of the BTMS message do we need this here - ////await Context.Bus.PublishDecisions(messageId, decisionResult, decisionContext, cancellationToken: cancellationToken); } else { @@ -87,7 +84,15 @@ public async Task OnHandle(AlvsClearanceRequest message, CancellationToken cance "Skipping Linking/Matching/Decisions for {Mrn} with MessageId {MessageId} with Pre-Processing Outcome {PreProcessingOutcome} Because Last AuditState was {AuditState}", message.Header?.EntryReference, messageId, preProcessingResult.Outcome.ToString(), preProcessingResult.Record.GetLatestAuditEntry().Status); + + if (preProcessingResult.Outcome == PreProcessingOutcome.Skipped || + preProcessingResult.Outcome == PreProcessingOutcome.AlreadyProcessed) + { + preProcessingResult.Record.Skipped(messageId, message.Version.GetValueOrDefault()); + } } + + await dbContext.SaveChangesAsync(Context.CancellationToken); } } diff --git a/Btms.Consumers/NotificationConsumer.cs b/Btms.Consumers/NotificationConsumer.cs index 93b8a05e..6d5f55f8 100644 --- a/Btms.Consumers/NotificationConsumer.cs +++ b/Btms.Consumers/NotificationConsumer.cs @@ -81,37 +81,22 @@ public async Task OnHandle(ImportNotification message, CancellationToken cancell var decisionResult = await decisionService.Process(decisionContext, Context.CancellationToken); await validationService.PostDecision(linkResult, decisionResult, Context.CancellationToken); - - - - - - await dbContext.SaveChangesAsync(Context.CancellationToken); - - - - ////await Context.Bus.PublishDecisions(messageId, decisionResult, decisionContext, cancellationToken: cancellationToken); } else if (preProcessingResult.IsDeleted()) { var linkContext = new ImportNotificationLinkContext(preProcessingResult.Record, preProcessingResult.ChangeSet); await linkingService.UnLink(linkContext, Context.CancellationToken); - await dbContext.SaveChangesAsync(Context.CancellationToken); } else { - if (preProcessingResult.Outcome != PreProcessingOutcome.Skipped || - preProcessingResult.Outcome != PreProcessingOutcome.AlreadyProcessed) - { - await dbContext.SaveChangesAsync(Context.CancellationToken); - } - logger.LogWarning("Skipping Linking/Matching/Decisions for {Id} with MessageId {MessageId} with Pre-Processing Outcome {PreProcessingOutcome} Because Last AuditState was {AuditState}", message.ReferenceNumber, messageId, preProcessingResult.Outcome.ToString(), preProcessingResult.Record.GetLatestAuditEntry().Status); LogStatus("IsCreatedOrChanged=false", message); } + await dbContext.SaveChangesAsync(Context.CancellationToken); + } } From 9bb9b956ee880a8917e10dfe0c9a5e17d1075893 Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Wed, 12 Feb 2025 06:18:46 +0000 Subject: [PATCH 10/13] removed unrequired code --- Btms.Consumers/AlvsClearanceRequestConsumer.cs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Btms.Consumers/AlvsClearanceRequestConsumer.cs b/Btms.Consumers/AlvsClearanceRequestConsumer.cs index 85942d67..d7b4ed23 100644 --- a/Btms.Consumers/AlvsClearanceRequestConsumer.cs +++ b/Btms.Consumers/AlvsClearanceRequestConsumer.cs @@ -84,12 +84,6 @@ public async Task OnHandle(AlvsClearanceRequest message, CancellationToken cance "Skipping Linking/Matching/Decisions for {Mrn} with MessageId {MessageId} with Pre-Processing Outcome {PreProcessingOutcome} Because Last AuditState was {AuditState}", message.Header?.EntryReference, messageId, preProcessingResult.Outcome.ToString(), preProcessingResult.Record.GetLatestAuditEntry().Status); - - if (preProcessingResult.Outcome == PreProcessingOutcome.Skipped || - preProcessingResult.Outcome == PreProcessingOutcome.AlreadyProcessed) - { - preProcessingResult.Record.Skipped(messageId, message.Version.GetValueOrDefault()); - } } await dbContext.SaveChangesAsync(Context.CancellationToken); From 5dc81ae05b0853db71a43b2434f9c12426198339 Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Wed, 12 Feb 2025 06:48:58 +0000 Subject: [PATCH 11/13] added unique index for decision number --- Btms.Backend.Data/Mongo/MongoIndexService.cs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/Btms.Backend.Data/Mongo/MongoIndexService.cs b/Btms.Backend.Data/Mongo/MongoIndexService.cs index 05fa10dc..b340fd78 100644 --- a/Btms.Backend.Data/Mongo/MongoIndexService.cs +++ b/Btms.Backend.Data/Mongo/MongoIndexService.cs @@ -12,18 +12,21 @@ public Task StartAsync(CancellationToken cancellationToken) { return Task.WhenAll( CreateIndex("MatchReferenceIdx", - Builders.IndexKeys.Ascending(n => n._MatchReference), cancellationToken), + Builders.IndexKeys.Ascending(n => n._MatchReference), cancellationToken: cancellationToken), CreateIndex("Created", - Builders.IndexKeys.Ascending(n => n.Created), cancellationToken), + Builders.IndexKeys.Ascending(n => n.Created), cancellationToken: cancellationToken), CreateIndex("CreatedSource", - Builders.IndexKeys.Ascending(n => n.CreatedSource), cancellationToken), + Builders.IndexKeys.Ascending(n => n.CreatedSource), cancellationToken: cancellationToken), CreateIndex("MatchReferenceIdx", - Builders.IndexKeys.Ascending(m => m._MatchReferences), cancellationToken), + Builders.IndexKeys.Ascending(m => m._MatchReferences), cancellationToken: cancellationToken), CreateIndex("Created", - Builders.IndexKeys.Ascending(m => m.Created), cancellationToken), + Builders.IndexKeys.Ascending(m => m.Created), cancellationToken: cancellationToken), CreateIndex("CreatedSource", - Builders.IndexKeys.Ascending(m => m.CreatedSource), cancellationToken) + Builders.IndexKeys.Ascending(m => m.CreatedSource), cancellationToken: cancellationToken), + CreateIndex("UniqueDecisionNumber", + Builders.IndexKeys.Ascending(m => m.EntryReference) + .Ascending(new StringFieldDefinition("decisions.header.decisionNumber")), true, cancellationToken: cancellationToken) ); @@ -34,7 +37,7 @@ public Task StopAsync(CancellationToken cancellationToken) return Task.CompletedTask; } - private async Task CreateIndex(string name, IndexKeysDefinition keys, CancellationToken cancellationToken) + private async Task CreateIndex(string name, IndexKeysDefinition keys, bool unique = false, CancellationToken cancellationToken = default) { try { @@ -43,6 +46,7 @@ private async Task CreateIndex(string name, IndexKeysDefinition keys, Canc { Name = name, Background = true, + Unique = unique, }); await database.GetCollection(typeof(T).Name).Indexes.CreateOneAsync(indexModel, cancellationToken: cancellationToken); } From 1d9c86c5fb8493c85f987eedead5ee2ac68389bc Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Wed, 12 Feb 2025 11:46:38 +0000 Subject: [PATCH 12/13] merge issues --- .../Services/Decisions/NoMatchDecisionsTest.cs | 6 +++--- Btms.Business/Services/Decisions/DecisionContext.cs | 4 ++-- Btms.Consumers/AlvsClearanceRequestConsumer.cs | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs b/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs index 63a08d74..c8d12d6c 100644 --- a/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs +++ b/Btms.Business.Tests/Services/Decisions/NoMatchDecisionsTest.cs @@ -36,7 +36,7 @@ public async Task WhenClearanceRequest_HasNotMatch_AndH220Checks_ThenNoDecisionS matchingResult.AddDocumentNoMatch(movement.Id!, movement.Items[0].ItemNumber!.Value, movement.Items[0].Documents?[0].DocumentReference!); // Act - var decisionResult = await sut.Process(new DecisionContext(new List(), [movement], matchingResult, "TestMessageId", true), CancellationToken.None); + var decisionResult = await sut.Process(new DecisionContext(new List(), [movement], matchingResult, "TestMessageId"), CancellationToken.None); // Assert decisionResult.Should().NotBeNull(); @@ -63,7 +63,7 @@ public async Task WhenClearanceRequest_HasNotMatch_AndNoChecks_ThenNoDecisionSho matchingResult.AddDocumentNoMatch(movements[0].Id!, movements[0].Items[0].ItemNumber!.Value, movements[0].Items[0].Documents?[0].DocumentReference!); // Act - var decisionResult = await sut.Process(new DecisionContext(new List(), movements, matchingResult, "TestMessageId", true), CancellationToken.None); + var decisionResult = await sut.Process(new DecisionContext(new List(), movements, matchingResult, "TestMessageId"), CancellationToken.None); // Assert decisionResult.Should().NotBeNull(); @@ -88,7 +88,7 @@ public async Task WhenClearanceRequest_HasNotMatch_ThenDecisionCodeShouldBeNoMat matchingResult.AddDocumentNoMatch(movements[0].Id!, movements[0].Items[0].ItemNumber!.Value, movements[0].Items[0].Documents?[0].DocumentReference!); // Act - var decisionResult = await sut.Process(new DecisionContext(new List(), movements, matchingResult, "TestMessageId", true), CancellationToken.None); + var decisionResult = await sut.Process(new DecisionContext(new List(), movements, matchingResult, "TestMessageId"), CancellationToken.None); // Assert decisionResult.Should().NotBeNull(); diff --git a/Btms.Business/Services/Decisions/DecisionContext.cs b/Btms.Business/Services/Decisions/DecisionContext.cs index df858853..fe3cb4b4 100644 --- a/Btms.Business/Services/Decisions/DecisionContext.cs +++ b/Btms.Business/Services/Decisions/DecisionContext.cs @@ -7,8 +7,8 @@ namespace Btms.Business.Services.Decisions; public class DecisionContext( List notifications, List movements, - MatchingResult matchingResult) - string messageId, + MatchingResult matchingResult, + string messageId) { public List Notifications { get; } = notifications; public List Movements { get; } = movements; diff --git a/Btms.Consumers/AlvsClearanceRequestConsumer.cs b/Btms.Consumers/AlvsClearanceRequestConsumer.cs index d7b4ed23..1de1a567 100644 --- a/Btms.Consumers/AlvsClearanceRequestConsumer.cs +++ b/Btms.Consumers/AlvsClearanceRequestConsumer.cs @@ -71,7 +71,7 @@ public async Task OnHandle(AlvsClearanceRequest message, CancellationToken cance new MatchingContext(linkResult.Notifications, linkResult.Movements), Context.CancellationToken); var decisionContext = - new DecisionContext(linkResult.Notifications, linkResult.Movements, matchResult, messageId, true); + new DecisionContext(linkResult.Notifications, linkResult.Movements, matchResult, messageId); var decisionResult = await decisionService.Process(decisionContext, Context.CancellationToken); await validationService.PostDecision(linkResult, decisionResult, Context.CancellationToken); From 4215d5497f92a11e3f3d76ee7e7a56fb0cd5380f Mon Sep 17 00:00:00 2001 From: Thomas Anderson Date: Wed, 12 Feb 2025 13:27:10 +0000 Subject: [PATCH 13/13] set the enum as strings on the changeset --- Btms.Model/ChangeLog/ChangeSet.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Btms.Model/ChangeLog/ChangeSet.cs b/Btms.Model/ChangeLog/ChangeSet.cs index ba06f0a3..6c920053 100644 --- a/Btms.Model/ChangeLog/ChangeSet.cs +++ b/Btms.Model/ChangeLog/ChangeSet.cs @@ -13,7 +13,8 @@ public class ChangeSet(JsonPatch jsonPatch, JsonNode jsonNodePrevious) { TypeInfoResolver = new ChangeSetTypeInfoResolver(), PropertyNameCaseInsensitive = true, - NumberHandling = JsonNumberHandling.AllowReadingFromString + NumberHandling = JsonNumberHandling.AllowReadingFromString, + Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) } }; public JsonPatch JsonPatch { get; } = jsonPatch;