CDMS-155 allows finalisedOnly to be passed to analytics endpoints to …

…filter on only those that have been finalised
DEFRA · Jan 28, 2025 · 32a0141 · 32a0141
1 parent d41fe63
commit 32a0141
Show file tree

Hide file tree

Showing 13 changed files with 100 additions and 62 deletions.
diff --git a/Btms.Analytics.Tests/FinalisedOnlyTests.cs b/Btms.Analytics.Tests/FinalisedOnlyTests.cs
@@ -0,0 +1,37 @@
+using Btms.Common.Extensions;
+using FluentAssertions;
+using Xunit;
+using Xunit.Abstractions;
+
+using Btms.Analytics.Tests.Fixtures;
+using Btms.Analytics.Tests.Helpers;
+using TestDataGenerator.Config;
+using TestDataGenerator.Scenarios;
+using TestDataGenerator.Scenarios.SpecificFiles;
+using TestGenerator.IntegrationTesting.Backend;
+using TestGenerator.IntegrationTesting.Backend.Extensions;
+using TestGenerator.IntegrationTesting.Backend.Fixtures;
+
+namespace Btms.Analytics.Tests;
+
+public class FinalisedOnlyTests(ITestOutputHelper output) : MultipleScenarioGeneratorBaseTest(output)
+{
+    [Theory]
+    [InlineData(typeof(Mrn24GBDDJER3ZFRMZAR9ScenarioGenerator), false, true)]
+    [InlineData(typeof(Mrn24GBDDJER3ZFRMZAR9ScenarioGenerator), true, false)]
+    public async Task ShouldReturnCorrectAggregation(Type generatorType, bool finalisedOnly, bool returnsResults)
+    {
+        EnsureEnvironmentInitialised(generatorType);
+
+        var result = await Client
+            .GetAnalyticsDashboard(["decisionsByDecisionCode"],
+                dateFrom: DateTime.MinValue, dateTo: DateTime.MaxValue,
+                finalisedOnly: finalisedOnly);
+
+        var chart = await result
+            .AnalyticsChartAs<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>>("decisionsByDecisionCode")!;
+
+        chart.Summary.Values.Count
+            .Should().Be(returnsResults ? 1 : 0);
+    }
+}
diff --git a/Btms.Analytics.Tests/ImportNotificationsByMaxVersionTests.cs b/Btms.Analytics.Tests/ImportNotificationsByMaxVersionTests.cs
@@ -19,7 +19,7 @@ public async Task WhenCalledLastMonth_ReturnExpectedAggregation()
     {
         TestOutputHelper.WriteLine("Querying for aggregated data");
         var result = (await GetImportNotificationsAggregationService()
-            .ByMaxVersion(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow()));
+            .ByMaxVersion(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow(), false));
 
         TestOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count);
 
@@ -31,7 +31,7 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
     {
         TestOutputHelper.WriteLine("Querying for aggregated data");
         var result = await ImportNotificationsAggregationService
-            .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour());
+            .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), false);
 
         TestOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
 
@@ -43,7 +43,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati
     {
         TestOutputHelper.WriteLine("Querying for aggregated data");
         var result = await ImportNotificationsAggregationService
-            .ByMaxVersion(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue);
+            .ByMaxVersion(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue, false);
 
         TestOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
 
@@ -55,7 +55,7 @@ public async Task WhenCalledWithChedType_ReturnsResults()
     {
         TestOutputHelper.WriteLine("Querying for aggregated data");
         var result = await ImportNotificationsAggregationService
-            .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), chedTypes: [ImportNotificationTypeEnum.Cveda]);
+            .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), false, chedTypes: [ImportNotificationTypeEnum.Cveda]);
 
         TestOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
 
@@ -67,7 +67,7 @@ public async Task WhenCalledWithCountry_ReturnsResults()
     {
         TestOutputHelper.WriteLine("Querying for aggregated data");
         var result = await ImportNotificationsAggregationService
-            .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), country: "ES");
+            .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), false, country: "ES");
 
         TestOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
 

diff --git a/Btms.Analytics.Tests/MovementsByDecisionsTests.cs b/Btms.Analytics.Tests/MovementsByDecisionsTests.cs
@@ -14,12 +14,11 @@ public class MovementsByDecisionsTests(ITestOutputHelper output)
     : ScenarioDatasetBaseTest(output, Datasets.FunctionalAnalyticsDecisionsDatasetName)
 {
     [Fact]
-    // [Fact(Skip = "Needs revisiting - needs more assertions, perhaps switch to individual scenario test")]
     public async Task WhenCalled_ReturnExpectedAggregation()
     {
         TestOutputHelper.WriteLine("Querying for aggregated data");
         var result = await MovementsAggregationService
-            .ByDecision(DateTime.MinValue, DateTime.MaxValue)!;
+            .ByDecision(DateTime.MinValue, DateTime.MaxValue, false)!;
 
         TestOutputHelper.WriteLine("{0} aggregated items found", result!.Result.Count());
 

diff --git a/Btms.Analytics/Extensions/ImportNotificationExtensions.cs b/Btms.Analytics/Extensions/ImportNotificationExtensions.cs
@@ -6,6 +6,7 @@ namespace Btms.Analytics.Extensions;
 public static class ImportNotificationExtensions
 {   
     public static IQueryable<ImportNotification> WhereFilteredByCreatedDateAndParams(this IQueryable<ImportNotification> source, DateTime from, DateTime to,
+        bool finalisedOnly = true,
         ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
     {
         return source

diff --git a/Btms.Analytics/Extensions/MovementExtensions.cs b/Btms.Analytics/Extensions/MovementExtensions.cs
@@ -6,29 +6,20 @@ namespace Btms.Analytics.Extensions;
 
 public static class MovementExtensions
 {   
-    public static IQueryable<Movement> WhereFilteredByCreatedDateAndParams(this IQueryable<Movement> source, DateTime from, DateTime to,
+    public static IQueryable<Movement>  WhereFilteredByCreatedDateAndParams(this IQueryable<Movement> source, DateTime from, DateTime to,
+        bool finalisedOnly,
         ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
     {
         return source
             .Where(m => (m.CreatedSource >= from && m.CreatedSource < to)
                         && (country == null || m.DispatchCountryCode == country)
+                        && (!finalisedOnly || m.Finalised.HasValue)
                         && (chedTypes == null || !chedTypes!.Any() ||
                             !m.BtmsStatus.ChedTypes!.Any() ||
                             m.BtmsStatus.ChedTypes!.Any(c => chedTypes!.Contains(c))));
 
     }
-    // public static IQueryable<Movement> WhereFilteredByCreatedDateAndParams(this IQueryable<Movement> source, DateTime from, DateTime to,
-    //     ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
-    // {
-    //     return source
-    //         .Where(m => (m.CreatedSource >= from && m.CreatedSource < to)
-    //                     && (country == null || m.DispatchCountryCode == country)
-    //                     && (chedTypes == null || !chedTypes!.Any() ||
-    //                         !m.BtmsStatus.ChedTypes!.Any() ||
-    //                         m.BtmsStatus.ChedTypes!.Any(c => chedTypes!.Contains(c))));
-    //
-    // }
-
+
     public class MovementWithLinkStatus
     {
         public required Movement Movement;

diff --git a/Btms.Analytics/IImportNotificationsAggregationService.cs b/Btms.Analytics/IImportNotificationsAggregationService.cs
@@ -8,6 +8,6 @@ public interface IImportNotificationsAggregationService
     public Task<MultiSeriesDatetimeDataset> ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
     public Task<SingleSeriesDataset> ByStatus(DateTime? from = null, DateTime? to = null);
     public Task<MultiSeriesDataset<ByNumericDimensionResult>> ByCommodityCount(DateTime from, DateTime to);
-    public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
+    public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
     public EntityDataset<ScenarioItem> Scenarios(DateTime? from, DateTime? to);
 }
diff --git a/Btms.Analytics/IMovementsAggregationService.cs b/Btms.Analytics/IMovementsAggregationService.cs
@@ -8,16 +8,16 @@ public interface IMovementsAggregationService
     public Task<MultiSeriesDatetimeDataset> ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
     public Task<SingleSeriesDataset> ByStatus(DateTime? from = null, DateTime? to = null);
     public Task<MultiSeriesDataset<ByNumericDimensionResult>> ByItemCount(DateTime from, DateTime to);
-    public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>> ByDecision(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
+    public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>> ByDecision(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
     // public Task<TabularDataset<ByNameDimensionResult>> ByDecisionAndLinkStatus(DateTime from, DateTime to);
     public Task<MultiSeriesDataset<ByNumericDimensionResult>> ByUniqueDocumentReferenceCount(DateTime from, DateTime to);
     public Task<SingleSeriesDataset> UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to);
     // public Task<MultiSeriesDataset> ByCheck(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
     public Task<EntityDataset<AuditHistory>?> GetHistory(string movementId);
-    public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
-    public Task<SingleSeriesDataset> ByMaxDecisionNumber(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
-    public Task<List<ExceptionResult>> GetExceptions(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
-    public Task<SingleSeriesDataset> ExceptionSummary(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
+    public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
+    public Task<SingleSeriesDataset> ByMaxDecisionNumber(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
+    public Task<List<ExceptionResult>> GetExceptions(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
+    public Task<SingleSeriesDataset> ExceptionSummary(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
     public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>> BySegment(DateTime from,
-        DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
+        DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
 }
diff --git a/Btms.Analytics/ImportNotificationsAggregationService.cs b/Btms.Analytics/ImportNotificationsAggregationService.cs
@@ -161,11 +161,14 @@ public Task<MultiSeriesDataset<ByNumericDimensionResult>> ByCommodityCount(DateT
         });
     }
 
-    public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
+    public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
     {
+        // TODO : At the moment this doesn't filter on finalisedOnly as thats not stored anywhere on the notification
+        // we'd need to denormalise the field, perhaps onto the relationship, to allow this filtering.
+
         var data = context
             .Notifications
-            .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country)
+            .WhereFilteredByCreatedDateAndParams(from, to, finalisedOnly, chedTypes, country)
             .GroupBy(n => new { MaxVersion =
                 n.AuditEntries.Where(a => a.CreatedBy == CreatedBySystem.Ipaffs).Max(a => a.Version )
             })

diff --git a/Btms.Analytics/MovementExceptions.cs b/Btms.Analytics/MovementExceptions.cs
@@ -11,14 +11,14 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger)
     //Returns a summary of the exceptions or a list
     // Means we can share the same anonymous / query code without needing to create loads
     // of classes
-    public (SingleSeriesDataset summary, List<ExceptionResult>) GetAllExceptions(DateTime from, DateTime to, bool summary, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
+    public (SingleSeriesDataset summary, List<ExceptionResult>) GetAllExceptions(DateTime from, DateTime to, bool finalisedOnly, bool summary, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
     {
         var exceptionsSummary = new SingleSeriesDataset();
         var exceptionsResult = new List<ExceptionResult>();
 
         var simplifiedMovementView = context
             .Movements
-            .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country)
+            .WhereFilteredByCreatedDateAndParams(from, to, finalisedOnly, chedTypes, country)
             .Where(m => m.BtmsStatus.LinkStatus != LinkStatusEnum.AllLinked)
             .Select(m => new
             {

diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs
@@ -199,11 +199,11 @@ public Task<SingleSeriesDataset> UniqueDocumentReferenceByMovementCount(DateTime
         return new EntityDataset<AuditHistory>(entries);
     }
 
-    public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
+    public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
     {
         var data = context
             .Movements
-            .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country)
+            .WhereFilteredByCreatedDateAndParams(from, to, finalisedOnly, chedTypes, country)
             .GroupBy(n => new { MaxVersion =
                 n.ClearanceRequests.Max(a => a.Header!.EntryVersionNumber )
             })
@@ -216,11 +216,11 @@ public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, Import
         });
     }
 
-    public Task<SingleSeriesDataset> ByMaxDecisionNumber(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
+    public Task<SingleSeriesDataset> ByMaxDecisionNumber(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
     {
         var data = context
             .Movements
-            .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country)
+            .WhereFilteredByCreatedDateAndParams(from, to, finalisedOnly, chedTypes, country)
             .GroupBy(n => new { MaxVersion =
                 n.Decisions.Max(a => a.Header!.DecisionNumber )
             })
@@ -233,20 +233,20 @@ public Task<SingleSeriesDataset> ByMaxDecisionNumber(DateTime from, DateTime to,
         });
     }
 
-    public Task<List<ExceptionResult>> GetExceptions(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
+    public Task<List<ExceptionResult>> GetExceptions(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
     {
         var movementExceptions = new MovementExceptions(context, logger);
         var (_, result) = movementExceptions
-            .GetAllExceptions(from, to, false, chedTypes, country);
+            .GetAllExceptions(from, to, finalisedOnly, false, chedTypes, country);
 
         return Task.FromResult(result);
     }
 
-    public Task<SingleSeriesDataset> ExceptionSummary(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
+    public Task<SingleSeriesDataset> ExceptionSummary(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
     {
         var movementExceptions = new MovementExceptions(context, logger);
         var (summary, _) = movementExceptions
-            .GetAllExceptions(from, to, true, chedTypes, country);
+            .GetAllExceptions(from, to, finalisedOnly, true, chedTypes, country);
 
         return Task.FromResult(summary);
     }
@@ -283,15 +283,16 @@ private Task<MultiSeriesDatetimeDataset> Aggregate(DateTime[] dateRange, Func<Bs
     /// </summary>
     /// <param name="from"></param>
     /// <param name="to"></param>
+    /// <param name="finalisedOnly"></param>
     /// <param name="chedTypes"></param>
     /// <param name="country"></param>
     /// <returns></returns>
     public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>> ByDecision(DateTime from,
-        DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
+        DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
     {
         var mongoQuery = context
             .Movements
-            .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country)
+            .WhereFilteredByCreatedDateAndParams(from, to, finalisedOnly, chedTypes, country)
             .SelectMany(d => d
                 .AlvsDecisionStatus.Context.DecisionComparison!.Checks
                 .Select(c => new
@@ -367,11 +368,11 @@ public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>>
     }
 
     public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>> BySegment(DateTime from,
-        DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
+        DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
     {
         var mongoQuery = context
             .Movements
-            .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country)
+            .WhereFilteredByCreatedDateAndParams(from, to, finalisedOnly, chedTypes, country)
             .Select(m => new
             {
                 DecisionStatus = m.AlvsDecisionStatus.Context.DecisionComparison == null ?
@@ -383,7 +384,8 @@ public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>>
             })
             .GroupBy(d => new
             {
-                Segment = d.BtmsStatus.Segment ?? MovementSegmentEnum.None,
+                d.BtmsStatus.Segment,
+                // Segment = d.BtmsStatus.Segment ?? MovementSegmentEnum.None,
                 d.BtmsStatus.LinkStatus,
                 d.BtmsStatus.Status,
                 d.DecisionStatus,
@@ -409,7 +411,7 @@ public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>>
             .OrderBy(s => s.Key)
             // .Where(g => g)
             .ToDictionary(
-                g => enumLookup.GetValue(g.Key),
+                g => enumLookup.GetValue(g.Key ?? MovementSegmentEnum.None),
                 g => g.Sum
             );
 
@@ -426,7 +428,7 @@ public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>>
                     Fields = new Dictionary<string, string>()
                     {
                         // { "Classification", enumLookup.GetValue(a.Key!.Segment) },
-                        { "Classification", enumLookup.GetValue(a.Key!.Segment!) },
+                        { "Classification", enumLookup.GetValue(a.Key.Segment ?? MovementSegmentEnum.None) },
                         // { "CheckCode", a.Key.CheckCode! },
                         // { "AlvsDecisionCode", a.Key.AlvsDecisionCode! },
                         // { "BtmsDecisionCode", a.Key.BtmsDecisionCode! }