From 32a0141970456d2e962b8c533ae44d3954c5cdfc Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Mon, 27 Jan 2025 17:51:07 +0000 Subject: [PATCH 1/4] CDMS-155 allows finalisedOnly to be passed to analytics endpoints to filter on only those that have been finalised --- Btms.Analytics.Tests/FinalisedOnlyTests.cs | 37 +++++++++++++++++++ .../ImportNotificationsByMaxVersionTests.cs | 10 ++--- .../MovementsByDecisionsTests.cs | 3 +- .../ImportNotificationExtensions.cs | 1 + .../Extensions/MovementExtensions.cs | 17 ++------- .../IImportNotificationsAggregationService.cs | 2 +- .../IMovementsAggregationService.cs | 12 +++--- .../ImportNotificationsAggregationService.cs | 7 +++- Btms.Analytics/MovementExceptions.cs | 4 +- Btms.Analytics/MovementsAggregationService.cs | 32 ++++++++-------- Btms.Backend/Config/AnalyticsDashboards.cs | 21 ++++++----- Btms.Backend/Endpoints/AnalyticsEndpoints.cs | 10 +++-- .../BtmsClient.cs | 6 ++- 13 files changed, 100 insertions(+), 62 deletions(-) create mode 100644 Btms.Analytics.Tests/FinalisedOnlyTests.cs diff --git a/Btms.Analytics.Tests/FinalisedOnlyTests.cs b/Btms.Analytics.Tests/FinalisedOnlyTests.cs new file mode 100644 index 00000000..0199d5ee --- /dev/null +++ b/Btms.Analytics.Tests/FinalisedOnlyTests.cs @@ -0,0 +1,37 @@ +using Btms.Common.Extensions; +using FluentAssertions; +using Xunit; +using Xunit.Abstractions; + +using Btms.Analytics.Tests.Fixtures; +using Btms.Analytics.Tests.Helpers; +using TestDataGenerator.Config; +using TestDataGenerator.Scenarios; +using TestDataGenerator.Scenarios.SpecificFiles; +using TestGenerator.IntegrationTesting.Backend; +using TestGenerator.IntegrationTesting.Backend.Extensions; +using TestGenerator.IntegrationTesting.Backend.Fixtures; + +namespace Btms.Analytics.Tests; + +public class FinalisedOnlyTests(ITestOutputHelper output) : MultipleScenarioGeneratorBaseTest(output) +{ + [Theory] + [InlineData(typeof(Mrn24GBDDJER3ZFRMZAR9ScenarioGenerator), false, true)] + [InlineData(typeof(Mrn24GBDDJER3ZFRMZAR9ScenarioGenerator), true, false)] + public async Task ShouldReturnCorrectAggregation(Type generatorType, bool finalisedOnly, bool returnsResults) + { + EnsureEnvironmentInitialised(generatorType); + + var result = await Client + .GetAnalyticsDashboard(["decisionsByDecisionCode"], + dateFrom: DateTime.MinValue, dateTo: DateTime.MaxValue, + finalisedOnly: finalisedOnly); + + var chart = await result + .AnalyticsChartAs>("decisionsByDecisionCode")!; + + chart.Summary.Values.Count + .Should().Be(returnsResults ? 1 : 0); + } +} \ No newline at end of file diff --git a/Btms.Analytics.Tests/ImportNotificationsByMaxVersionTests.cs b/Btms.Analytics.Tests/ImportNotificationsByMaxVersionTests.cs index 1aa1faa5..1a33ec5a 100644 --- a/Btms.Analytics.Tests/ImportNotificationsByMaxVersionTests.cs +++ b/Btms.Analytics.Tests/ImportNotificationsByMaxVersionTests.cs @@ -19,7 +19,7 @@ public async Task WhenCalledLastMonth_ReturnExpectedAggregation() { TestOutputHelper.WriteLine("Querying for aggregated data"); var result = (await GetImportNotificationsAggregationService() - .ByMaxVersion(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow())); + .ByMaxVersion(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow(), false)); TestOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count); @@ -31,7 +31,7 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation() { TestOutputHelper.WriteLine("Querying for aggregated data"); var result = await ImportNotificationsAggregationService - .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour()); + .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), false); TestOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); @@ -43,7 +43,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati { TestOutputHelper.WriteLine("Querying for aggregated data"); var result = await ImportNotificationsAggregationService - .ByMaxVersion(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue); + .ByMaxVersion(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue, false); TestOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); @@ -55,7 +55,7 @@ public async Task WhenCalledWithChedType_ReturnsResults() { TestOutputHelper.WriteLine("Querying for aggregated data"); var result = await ImportNotificationsAggregationService - .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), chedTypes: [ImportNotificationTypeEnum.Cveda]); + .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), false, chedTypes: [ImportNotificationTypeEnum.Cveda]); TestOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); @@ -67,7 +67,7 @@ public async Task WhenCalledWithCountry_ReturnsResults() { TestOutputHelper.WriteLine("Querying for aggregated data"); var result = await ImportNotificationsAggregationService - .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), country: "ES"); + .ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), false, country: "ES"); TestOutputHelper.WriteLine($"{result.Values.Count} aggregated items found"); diff --git a/Btms.Analytics.Tests/MovementsByDecisionsTests.cs b/Btms.Analytics.Tests/MovementsByDecisionsTests.cs index efb672e7..33ca7225 100644 --- a/Btms.Analytics.Tests/MovementsByDecisionsTests.cs +++ b/Btms.Analytics.Tests/MovementsByDecisionsTests.cs @@ -14,12 +14,11 @@ public class MovementsByDecisionsTests(ITestOutputHelper output) : ScenarioDatasetBaseTest(output, Datasets.FunctionalAnalyticsDecisionsDatasetName) { [Fact] - // [Fact(Skip = "Needs revisiting - needs more assertions, perhaps switch to individual scenario test")] public async Task WhenCalled_ReturnExpectedAggregation() { TestOutputHelper.WriteLine("Querying for aggregated data"); var result = await MovementsAggregationService - .ByDecision(DateTime.MinValue, DateTime.MaxValue)!; + .ByDecision(DateTime.MinValue, DateTime.MaxValue, false)!; TestOutputHelper.WriteLine("{0} aggregated items found", result!.Result.Count()); diff --git a/Btms.Analytics/Extensions/ImportNotificationExtensions.cs b/Btms.Analytics/Extensions/ImportNotificationExtensions.cs index 1cf7d2b5..c5dece5f 100644 --- a/Btms.Analytics/Extensions/ImportNotificationExtensions.cs +++ b/Btms.Analytics/Extensions/ImportNotificationExtensions.cs @@ -6,6 +6,7 @@ namespace Btms.Analytics.Extensions; public static class ImportNotificationExtensions { public static IQueryable WhereFilteredByCreatedDateAndParams(this IQueryable source, DateTime from, DateTime to, + bool finalisedOnly = true, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) { return source diff --git a/Btms.Analytics/Extensions/MovementExtensions.cs b/Btms.Analytics/Extensions/MovementExtensions.cs index 8e8c78d9..04807315 100644 --- a/Btms.Analytics/Extensions/MovementExtensions.cs +++ b/Btms.Analytics/Extensions/MovementExtensions.cs @@ -6,29 +6,20 @@ namespace Btms.Analytics.Extensions; public static class MovementExtensions { - public static IQueryable WhereFilteredByCreatedDateAndParams(this IQueryable source, DateTime from, DateTime to, + public static IQueryable WhereFilteredByCreatedDateAndParams(this IQueryable source, DateTime from, DateTime to, + bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) { return source .Where(m => (m.CreatedSource >= from && m.CreatedSource < to) && (country == null || m.DispatchCountryCode == country) + && (!finalisedOnly || m.Finalised.HasValue) && (chedTypes == null || !chedTypes!.Any() || !m.BtmsStatus.ChedTypes!.Any() || m.BtmsStatus.ChedTypes!.Any(c => chedTypes!.Contains(c)))); } - // public static IQueryable WhereFilteredByCreatedDateAndParams(this IQueryable source, DateTime from, DateTime to, - // ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) - // { - // return source - // .Where(m => (m.CreatedSource >= from && m.CreatedSource < to) - // && (country == null || m.DispatchCountryCode == country) - // && (chedTypes == null || !chedTypes!.Any() || - // !m.BtmsStatus.ChedTypes!.Any() || - // m.BtmsStatus.ChedTypes!.Any(c => chedTypes!.Contains(c)))); - // - // } - + public class MovementWithLinkStatus { public required Movement Movement; diff --git a/Btms.Analytics/IImportNotificationsAggregationService.cs b/Btms.Analytics/IImportNotificationsAggregationService.cs index 62d92fa2..4df6447c 100644 --- a/Btms.Analytics/IImportNotificationsAggregationService.cs +++ b/Btms.Analytics/IImportNotificationsAggregationService.cs @@ -8,6 +8,6 @@ public interface IImportNotificationsAggregationService public Task ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); public Task ByStatus(DateTime? from = null, DateTime? to = null); public Task> ByCommodityCount(DateTime from, DateTime to); - public Task ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null); + public Task ByMaxVersion(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null); public EntityDataset Scenarios(DateTime? from, DateTime? to); } \ No newline at end of file diff --git a/Btms.Analytics/IMovementsAggregationService.cs b/Btms.Analytics/IMovementsAggregationService.cs index 6b5bc6d9..07ed78bf 100644 --- a/Btms.Analytics/IMovementsAggregationService.cs +++ b/Btms.Analytics/IMovementsAggregationService.cs @@ -8,16 +8,16 @@ public interface IMovementsAggregationService public Task ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day); public Task ByStatus(DateTime? from = null, DateTime? to = null); public Task> ByItemCount(DateTime from, DateTime to); - public Task> ByDecision(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null); + public Task> ByDecision(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null); // public Task> ByDecisionAndLinkStatus(DateTime from, DateTime to); public Task> ByUniqueDocumentReferenceCount(DateTime from, DateTime to); public Task UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to); // public Task ByCheck(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null); public Task?> GetHistory(string movementId); - public Task ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null); - public Task ByMaxDecisionNumber(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null); - public Task> GetExceptions(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null); - public Task ExceptionSummary(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null); + public Task ByMaxVersion(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null); + public Task ByMaxDecisionNumber(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null); + public Task> GetExceptions(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null); + public Task ExceptionSummary(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null); public Task> BySegment(DateTime from, - DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null); + DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null); } \ No newline at end of file diff --git a/Btms.Analytics/ImportNotificationsAggregationService.cs b/Btms.Analytics/ImportNotificationsAggregationService.cs index 7c4d1691..76a503de 100644 --- a/Btms.Analytics/ImportNotificationsAggregationService.cs +++ b/Btms.Analytics/ImportNotificationsAggregationService.cs @@ -161,11 +161,14 @@ public Task> ByCommodityCount(DateT }); } - public Task ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) + public Task ByMaxVersion(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) { + // TODO : At the moment this doesn't filter on finalisedOnly as thats not stored anywhere on the notification + // we'd need to denormalise the field, perhaps onto the relationship, to allow this filtering. + var data = context .Notifications - .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country) + .WhereFilteredByCreatedDateAndParams(from, to, finalisedOnly, chedTypes, country) .GroupBy(n => new { MaxVersion = n.AuditEntries.Where(a => a.CreatedBy == CreatedBySystem.Ipaffs).Max(a => a.Version ) }) diff --git a/Btms.Analytics/MovementExceptions.cs b/Btms.Analytics/MovementExceptions.cs index d06667b8..5d9b6369 100644 --- a/Btms.Analytics/MovementExceptions.cs +++ b/Btms.Analytics/MovementExceptions.cs @@ -11,14 +11,14 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger) //Returns a summary of the exceptions or a list // Means we can share the same anonymous / query code without needing to create loads // of classes - public (SingleSeriesDataset summary, List) GetAllExceptions(DateTime from, DateTime to, bool summary, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) + public (SingleSeriesDataset summary, List) GetAllExceptions(DateTime from, DateTime to, bool finalisedOnly, bool summary, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) { var exceptionsSummary = new SingleSeriesDataset(); var exceptionsResult = new List(); var simplifiedMovementView = context .Movements - .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country) + .WhereFilteredByCreatedDateAndParams(from, to, finalisedOnly, chedTypes, country) .Where(m => m.BtmsStatus.LinkStatus != LinkStatusEnum.AllLinked) .Select(m => new { diff --git a/Btms.Analytics/MovementsAggregationService.cs b/Btms.Analytics/MovementsAggregationService.cs index d9120d66..24bef0fa 100644 --- a/Btms.Analytics/MovementsAggregationService.cs +++ b/Btms.Analytics/MovementsAggregationService.cs @@ -199,11 +199,11 @@ public Task UniqueDocumentReferenceByMovementCount(DateTime return new EntityDataset(entries); } - public Task ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) + public Task ByMaxVersion(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) { var data = context .Movements - .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country) + .WhereFilteredByCreatedDateAndParams(from, to, finalisedOnly, chedTypes, country) .GroupBy(n => new { MaxVersion = n.ClearanceRequests.Max(a => a.Header!.EntryVersionNumber ) }) @@ -216,11 +216,11 @@ public Task ByMaxVersion(DateTime from, DateTime to, Import }); } - public Task ByMaxDecisionNumber(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) + public Task ByMaxDecisionNumber(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) { var data = context .Movements - .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country) + .WhereFilteredByCreatedDateAndParams(from, to, finalisedOnly, chedTypes, country) .GroupBy(n => new { MaxVersion = n.Decisions.Max(a => a.Header!.DecisionNumber ) }) @@ -233,20 +233,20 @@ public Task ByMaxDecisionNumber(DateTime from, DateTime to, }); } - public Task> GetExceptions(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) + public Task> GetExceptions(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) { var movementExceptions = new MovementExceptions(context, logger); var (_, result) = movementExceptions - .GetAllExceptions(from, to, false, chedTypes, country); + .GetAllExceptions(from, to, finalisedOnly, false, chedTypes, country); return Task.FromResult(result); } - public Task ExceptionSummary(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) + public Task ExceptionSummary(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) { var movementExceptions = new MovementExceptions(context, logger); var (summary, _) = movementExceptions - .GetAllExceptions(from, to, true, chedTypes, country); + .GetAllExceptions(from, to, finalisedOnly, true, chedTypes, country); return Task.FromResult(summary); } @@ -283,15 +283,16 @@ private Task Aggregate(DateTime[] dateRange, Func /// /// + /// /// /// /// public Task> ByDecision(DateTime from, - DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) + DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) { var mongoQuery = context .Movements - .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country) + .WhereFilteredByCreatedDateAndParams(from, to, finalisedOnly, chedTypes, country) .SelectMany(d => d .AlvsDecisionStatus.Context.DecisionComparison!.Checks .Select(c => new @@ -367,11 +368,11 @@ public Task> } public Task> BySegment(DateTime from, - DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) + DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null) { var mongoQuery = context .Movements - .WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country) + .WhereFilteredByCreatedDateAndParams(from, to, finalisedOnly, chedTypes, country) .Select(m => new { DecisionStatus = m.AlvsDecisionStatus.Context.DecisionComparison == null ? @@ -383,7 +384,8 @@ public Task> }) .GroupBy(d => new { - Segment = d.BtmsStatus.Segment ?? MovementSegmentEnum.None, + d.BtmsStatus.Segment, + // Segment = d.BtmsStatus.Segment ?? MovementSegmentEnum.None, d.BtmsStatus.LinkStatus, d.BtmsStatus.Status, d.DecisionStatus, @@ -409,7 +411,7 @@ public Task> .OrderBy(s => s.Key) // .Where(g => g) .ToDictionary( - g => enumLookup.GetValue(g.Key), + g => enumLookup.GetValue(g.Key ?? MovementSegmentEnum.None), g => g.Sum ); @@ -426,7 +428,7 @@ public Task> Fields = new Dictionary() { // { "Classification", enumLookup.GetValue(a.Key!.Segment) }, - { "Classification", enumLookup.GetValue(a.Key!.Segment!) }, + { "Classification", enumLookup.GetValue(a.Key.Segment ?? MovementSegmentEnum.None) }, // { "CheckCode", a.Key.CheckCode! }, // { "AlvsDecisionCode", a.Key.AlvsDecisionCode! }, // { "BtmsDecisionCode", a.Key.BtmsDecisionCode! } diff --git a/Btms.Backend/Config/AnalyticsDashboards.cs b/Btms.Backend/Config/AnalyticsDashboards.cs index 391e87a4..831a1737 100644 --- a/Btms.Backend/Config/AnalyticsDashboards.cs +++ b/Btms.Backend/Config/AnalyticsDashboards.cs @@ -17,7 +17,8 @@ public static async Task> GetCharts( ImportNotificationTypeEnum[] chedTypes, string? country, DateTime? dateFrom, - DateTime? dateTo + DateTime? dateTo, + bool finalisedOnly ) { var charts = new Dictionary>> @@ -98,27 +99,27 @@ public static async Task> GetCharts( }, { "decisionsByDecisionCode", - () => movementsService.ByDecision(dateFrom ?? DateTime.Today.MonthAgo(), dateTo ?? DateTime.Now, chedTypes, country).AsIDataset() + () => movementsService.ByDecision(dateFrom ?? DateTime.Today.MonthAgo(), dateTo ?? DateTime.Now, finalisedOnly, chedTypes, country).AsIDataset() + }, + { + "movementsBySegment", + () => movementsService.BySegment(dateFrom ?? DateTime.Today.MonthAgo(), dateTo ?? DateTime.Now, finalisedOnly, chedTypes, country).AsIDataset() }, - // { - // "movementsBySegment", - // () => movementsService.BySegment(dateFrom ?? DateTime.Today.MonthAgo(), dateTo ?? DateTime.Now, chedTypes, country).AsIDataset() - // }, { "importNotificationsByVersion", - () => importService.ByMaxVersion(dateFrom ?? DateTime.Today.AddMonths(-3), dateTo ?? DateTime.Today, chedTypes, country).AsIDataset() + () => importService.ByMaxVersion(dateFrom ?? DateTime.Today.AddMonths(-3), dateTo ?? DateTime.Today, finalisedOnly, chedTypes, country).AsIDataset() }, { "movementsByMaxEntryVersion", - () => movementsService.ByMaxVersion(dateFrom ?? DateTime.Today.AddMonths(-3), dateTo ?? DateTime.Today, chedTypes, country).AsIDataset() + () => movementsService.ByMaxVersion(dateFrom ?? DateTime.Today.AddMonths(-3), dateTo ?? DateTime.Today, finalisedOnly, chedTypes, country).AsIDataset() }, { "movementsByMaxDecisionNumber", - () => movementsService.ByMaxDecisionNumber(dateFrom ?? DateTime.Today.AddMonths(-3), dateTo ?? DateTime.Today, chedTypes, country).AsIDataset() + () => movementsService.ByMaxDecisionNumber(dateFrom ?? DateTime.Today.AddMonths(-3), dateTo ?? DateTime.Today, finalisedOnly, chedTypes, country).AsIDataset() }, { "movementsExceptions", - () => movementsService.ExceptionSummary(dateFrom ?? DateTime.Today.AddMonths(-3), dateTo ?? DateTime.Today, chedTypes, country).AsIDataset() + () => movementsService.ExceptionSummary(dateFrom ?? DateTime.Today.AddMonths(-3), dateTo ?? DateTime.Today, finalisedOnly, chedTypes, country).AsIDataset() } }; diff --git a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs index 607a01ee..12c57b06 100644 --- a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs +++ b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs @@ -64,12 +64,13 @@ private static async Task Exceptions( [FromQuery(Name = "chedType")] ImportNotificationTypeEnum[] chedTypes, [FromQuery(Name = "country")] string? country, [FromQuery(Name = "dateFrom")] DateTime? dateFrom, - [FromQuery(Name = "dateTo")] DateTime? dateTo) + [FromQuery(Name = "dateTo")] DateTime? dateTo, + [FromQuery(Name = "finalisedOnly")] bool finalisedOnly = true) { var result = await movementsService .GetExceptions(dateFrom ?? DateTime.MinValue, dateTo ?? DateTime.Today, - chedTypes, country); + finalisedOnly, chedTypes, country); return result.HasValue() ? TypedResults.Json(result) : @@ -118,14 +119,15 @@ private static async Task GetDashboard( [FromQuery(Name = "chedType")] ImportNotificationTypeEnum[] chedTypes, [FromQuery(Name = "coo")] string? countryOfOrigin, [FromQuery(Name = "dateFrom")] DateTime? dateFrom, - [FromQuery(Name = "dateTo")] DateTime? dateTo) + [FromQuery(Name = "dateTo")] DateTime? dateTo, + [FromQuery(Name = "finalisedOnly")] bool finalisedOnly = true) { var logger = ApplicationLogging.CreateLogger("AnalyticsEndpoints"); var result = await AnalyticsDashboards .GetCharts(logger, importService, movementsService, chartsToRender, - chedTypes, countryOfOrigin, dateFrom, dateTo); + chedTypes, countryOfOrigin, dateFrom, dateTo, finalisedOnly); var options = new JsonSerializerOptions diff --git a/TestGenerator.IntegrationTesting.Backend/BtmsClient.cs b/TestGenerator.IntegrationTesting.Backend/BtmsClient.cs index f0fc2240..41d1e9e4 100644 --- a/TestGenerator.IntegrationTesting.Backend/BtmsClient.cs +++ b/TestGenerator.IntegrationTesting.Backend/BtmsClient.cs @@ -124,16 +124,18 @@ public Task GetAnalyticsDashboard(string[] charts, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null, DateTime? dateFrom = null, - DateTime? dateTo = null) + DateTime? dateTo = null, + bool finalisedOnly = false) { var chartsFilter = String.Join("&chartsToRender=", charts); var dateFromFilter = dateFrom.HasValue ? $"&dateFrom={dateFrom.Value:yyyy-MM-dd}" : ""; var dateToFilter = dateTo.HasValue ? $"&dateTo={dateTo.Value:yyyy-MM-dd}" : ""; var countryFilter = country != null ? $"&coo={country}" : ""; var chedTypeFilter = chedTypes != null ? "&chedType=" + String.Join("&chedType=", chedTypes) : ""; + var finalisedOnlyFilter = $"&finalisedOnly={finalisedOnly}"; return client.GetAsync( - $"/analytics/dashboard?chartsToRender={chartsFilter}{dateFromFilter}{dateToFilter}{countryFilter}{chedTypeFilter}"); + $"/analytics/dashboard?chartsToRender={chartsFilter}{dateFromFilter}{dateToFilter}{countryFilter}{chedTypeFilter}{finalisedOnlyFilter}"); } private async Task PostCommand(T command, string uri) From 01f9900df81b6b1f30d4d8cdc7fe61c7764aaf8e Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Tue, 28 Jan 2025 10:07:09 +0000 Subject: [PATCH 2/4] CDMS-155 attempting to fix sonarqube 'too many arguments' error --- Btms.Backend/Config/AnalyticsDashboards.cs | 106 +++++++++++++++++-- Btms.Backend/Endpoints/AnalyticsEndpoints.cs | 14 ++- Btms.Backend/Program.cs | 10 ++ 3 files changed, 115 insertions(+), 15 deletions(-) diff --git a/Btms.Backend/Config/AnalyticsDashboards.cs b/Btms.Backend/Config/AnalyticsDashboards.cs index 831a1737..f1973a41 100644 --- a/Btms.Backend/Config/AnalyticsDashboards.cs +++ b/Btms.Backend/Config/AnalyticsDashboards.cs @@ -3,10 +3,93 @@ using Btms.Common.Extensions; using Btms.Model.Ipaffs; using FluentAssertions; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.ModelBinding; +using Microsoft.AspNetCore.Mvc.ModelBinding.Binders; using MongoDB.Driver.Linq; namespace Btms.Backend.Config; + +[ModelBinder(BinderType = typeof(DateRangeBinder))] +public class DateRange { + [FromQuery(Name = "dateFrom")] + public DateTime? From { get; set; } + [FromQuery(Name = "dateTo")] + public DateTime? To { get; set; } + + public static DateRange Default() + { + return new DateRange() { From = DateTime.Now, To = DateTime.Now }; + } + public static bool TryParse(string query, out DateRange dateRange) + { + dateRange = new DateRange() { From = DateTime.Now, To = DateTime.Now }; + return true; + } +} + +public class DateRangeBinderProvider : IModelBinderProvider +{ + public DateRangeBinderProvider() + { + Console.WriteLine("Testing12"); + } + public IModelBinder GetBinder(ModelBinderProviderContext context) + { + if (context == null) + { + throw new ArgumentNullException(nameof(context)); + } + + if (context.Metadata.ModelType == typeof(DateRange)) + { + return new BinderTypeModelBinder(typeof(DateRangeBinder)); + } + + return new BinderTypeModelBinder(context.Metadata.ModelType); + // ; + // return null!; //new BinderTypeModelBinder(); + } +} + +public class DateRangeBinder: IModelBinder +{ + public DateRangeBinder() + { + Console.WriteLine("Testing12"); + } + public Task BindModelAsync(ModelBindingContext bindingContext) + { + if (bindingContext == null) + { + throw new ArgumentNullException(nameof(bindingContext)); + } + + var dateFrom = bindingContext.ValueProvider.GetValue("dateFrom").FirstValue; + var dateTo = bindingContext.ValueProvider.GetValue("dateTo").FirstValue; + + var dateRange = new DateRange() + { + From = DateTime.Now, To = DateTime.Now + }; + // var name = bindingContext.ValueProvider.GetValue("name").FirstValue; + // var beginName = bindingContext.ValueProvider.GetValue("name:contains").FirstValue; + // var exactName = bindingContext.ValueProvider.GetValue("name:exact").FirstValue; + // if (name.ToLower().Contains("tom")) { + // model1.name = name; + // } + // if (beginName.ToLower().StartsWith("tom")) { + // model1.beginName = beginName; + // } + // if (exactName.Contains("Tom")) { + // model1.exactName = exactName; + // } + bindingContext.Result = ModelBindingResult.Success(dateRange); + return Task.CompletedTask; + } +} + public static class AnalyticsDashboards { public static async Task> GetCharts( @@ -16,8 +99,9 @@ public static async Task> GetCharts( string[] chartsToRender, ImportNotificationTypeEnum[] chedTypes, string? country, - DateTime? dateFrom, - DateTime? dateTo, + DateRange dateRange, + // DateTime? dateFrom, + // DateTime? dateTo, bool finalisedOnly ) { @@ -75,11 +159,11 @@ bool finalisedOnly }, { "uniqueDocumentReferenceCount", - () => movementsService.ByUniqueDocumentReferenceCount(dateFrom ?? DateTime.Today.MonthAgo(), dateTo ?? DateTime.Now).AsIDataset() + () => movementsService.ByUniqueDocumentReferenceCount(dateRange.From ?? DateTime.Today.MonthAgo(), dateRange.To ?? DateTime.Now).AsIDataset() }, { "uniqueDocumentReferenceByMovementCount", - () => movementsService.UniqueDocumentReferenceByMovementCount(dateFrom ?? DateTime.Today.MonthAgo(), dateTo ?? DateTime.Now).AsIDataset() + () => movementsService.UniqueDocumentReferenceByMovementCount(dateRange.From ?? DateTime.Today.MonthAgo(), dateRange.To ?? DateTime.Now).AsIDataset() }, { "lastMonthMovementsByUniqueDocumentReferenceCount", @@ -87,7 +171,7 @@ bool finalisedOnly }, { "movementsByUniqueDocumentReferenceCount", - () => movementsService.ByUniqueDocumentReferenceCount(dateFrom ?? DateTime.Today.MonthAgo(), dateTo ?? DateTime.Now).AsIDataset() + () => movementsService.ByUniqueDocumentReferenceCount(dateRange.From ?? DateTime.Today.MonthAgo(), dateRange.To ?? DateTime.Now).AsIDataset() }, { "lastMonthUniqueDocumentReferenceByMovementCount", @@ -99,27 +183,27 @@ bool finalisedOnly }, { "decisionsByDecisionCode", - () => movementsService.ByDecision(dateFrom ?? DateTime.Today.MonthAgo(), dateTo ?? DateTime.Now, finalisedOnly, chedTypes, country).AsIDataset() + () => movementsService.ByDecision(dateRange.From ?? DateTime.Today.MonthAgo(), dateRange.To ?? DateTime.Now, finalisedOnly, chedTypes, country).AsIDataset() }, { "movementsBySegment", - () => movementsService.BySegment(dateFrom ?? DateTime.Today.MonthAgo(), dateTo ?? DateTime.Now, finalisedOnly, chedTypes, country).AsIDataset() + () => movementsService.BySegment(dateRange.From ?? DateTime.Today.MonthAgo(), dateRange.To ?? DateTime.Now, finalisedOnly, chedTypes, country).AsIDataset() }, { "importNotificationsByVersion", - () => importService.ByMaxVersion(dateFrom ?? DateTime.Today.AddMonths(-3), dateTo ?? DateTime.Today, finalisedOnly, chedTypes, country).AsIDataset() + () => importService.ByMaxVersion(dateRange.From ?? DateTime.Today.AddMonths(-3), dateRange.To ?? DateTime.Today, finalisedOnly, chedTypes, country).AsIDataset() }, { "movementsByMaxEntryVersion", - () => movementsService.ByMaxVersion(dateFrom ?? DateTime.Today.AddMonths(-3), dateTo ?? DateTime.Today, finalisedOnly, chedTypes, country).AsIDataset() + () => movementsService.ByMaxVersion(dateRange.From ?? DateTime.Today.AddMonths(-3), dateRange.To ?? DateTime.Today, finalisedOnly, chedTypes, country).AsIDataset() }, { "movementsByMaxDecisionNumber", - () => movementsService.ByMaxDecisionNumber(dateFrom ?? DateTime.Today.AddMonths(-3), dateTo ?? DateTime.Today, finalisedOnly, chedTypes, country).AsIDataset() + () => movementsService.ByMaxDecisionNumber(dateRange.From ?? DateTime.Today.AddMonths(-3), dateRange.To ?? DateTime.Today, finalisedOnly, chedTypes, country).AsIDataset() }, { "movementsExceptions", - () => movementsService.ExceptionSummary(dateFrom ?? DateTime.Today.AddMonths(-3), dateTo ?? DateTime.Today, finalisedOnly, chedTypes, country).AsIDataset() + () => movementsService.ExceptionSummary(dateRange.From ?? DateTime.Today.AddMonths(-3), dateRange.To ?? DateTime.Today, finalisedOnly, chedTypes, country).AsIDataset() } }; diff --git a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs index 12c57b06..2dafd8db 100644 --- a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs +++ b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs @@ -6,6 +6,7 @@ using Btms.Model.Extensions; using Btms.Model.Ipaffs; using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.ModelBinding; using Microsoft.Extensions.Options; namespace Btms.Backend.Endpoints; @@ -111,23 +112,28 @@ private static async Task RecordCurrentState( await importNotificationMetrics.RecordCurrentState(); return Results.Ok(); } - + private static async Task GetDashboard( [FromServices] IImportNotificationsAggregationService importService, [FromServices] IMovementsAggregationService movementsService, [FromQuery] string[] chartsToRender, [FromQuery(Name = "chedType")] ImportNotificationTypeEnum[] chedTypes, [FromQuery(Name = "coo")] string? countryOfOrigin, - [FromQuery(Name = "dateFrom")] DateTime? dateFrom, - [FromQuery(Name = "dateTo")] DateTime? dateTo, + // DateRange dateRange, + // [FromQuery] DateRange? dateRange = DateRange.Default, + [ModelBinder(typeof(DateRangeBinder))] DateRange dateRange, + // DateRange dateRange, + // [FromQuery(Name = "dateFrom")] DateTime? dateFrom, + // [FromQuery(Name = "dateTo")] DateTime? dateTo, [FromQuery(Name = "finalisedOnly")] bool finalisedOnly = true) { var logger = ApplicationLogging.CreateLogger("AnalyticsEndpoints"); + var result = await AnalyticsDashboards .GetCharts(logger, importService, movementsService, chartsToRender, - chedTypes, countryOfOrigin, dateFrom, dateTo, finalisedOnly); + chedTypes, countryOfOrigin, DateRange.Default(), finalisedOnly); var options = new JsonSerializerOptions diff --git a/Btms.Backend/Program.cs b/Btms.Backend/Program.cs index 5f3c1fec..76ce1eda 100644 --- a/Btms.Backend/Program.cs +++ b/Btms.Backend/Program.cs @@ -26,6 +26,7 @@ using JsonApiDotNetCore.Serialization.Response; using Microsoft.AspNetCore.Diagnostics.HealthChecks; using Microsoft.Extensions.Options; +// using Microsoft.AspNetCore.Mvc.ModelBinding.Binders; using OpenTelemetry; using OpenTelemetry.Metrics; using OpenTelemetry.Trace; @@ -102,6 +103,15 @@ static void ConfigureWebApplication(WebApplicationBuilder builder) builder.Services.AddBusinessServices(builder.Configuration); builder.Services.AddConsumers(builder.Configuration); + // var provider = new SimpleTypeModelBinderProvider( + // typeof(DateRange), new AnalyticsEndpoints.DateRangeBinder()); + // + // builder.Services.Insert(typeof(ModelBinderProvider), 0, provider); + builder.Services.AddControllers(options => + { + options.ModelBinderProviders.Insert(0, new DateRangeBinderProvider()); + }); + ConfigureEndpoints(builder); builder.Services.AddHttpClient(); From c25b858a6a67fced397166c84387613c58e12ab9 Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Tue, 28 Jan 2025 10:27:35 +0000 Subject: [PATCH 3/4] Using AsParams --- Btms.Backend/Endpoints/AnalyticsEndpoints.cs | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs index 2dafd8db..3f81aa0e 100644 --- a/Btms.Backend/Endpoints/AnalyticsEndpoints.cs +++ b/Btms.Backend/Endpoints/AnalyticsEndpoints.cs @@ -62,15 +62,14 @@ private static async Task Timeline( private static async Task Exceptions( [FromServices] IMovementsAggregationService movementsService, + [AsParameters] DateRange dateRange, [FromQuery(Name = "chedType")] ImportNotificationTypeEnum[] chedTypes, [FromQuery(Name = "country")] string? country, - [FromQuery(Name = "dateFrom")] DateTime? dateFrom, - [FromQuery(Name = "dateTo")] DateTime? dateTo, [FromQuery(Name = "finalisedOnly")] bool finalisedOnly = true) { var result = await movementsService - .GetExceptions(dateFrom ?? DateTime.MinValue, dateTo ?? DateTime.Today, + .GetExceptions(dateRange.From ?? DateTime.MinValue, dateRange.To ?? DateTime.Today, finalisedOnly, chedTypes, country); return result.HasValue() ? @@ -81,11 +80,10 @@ var result private static IResult Scenarios( [FromServices] IMovementsAggregationService movementsService, [FromServices] IImportNotificationsAggregationService importService, - [FromQuery(Name = "dateFrom")] DateTime? dateFrom, - [FromQuery(Name = "dateTo")] DateTime? dateTo) + [AsParameters] DateRange dateRange) { var result - = importService.Scenarios(dateFrom, dateTo); + = importService.Scenarios(dateRange.From, dateRange.To); if (result.HasValue()) { @@ -116,15 +114,10 @@ private static async Task RecordCurrentState( private static async Task GetDashboard( [FromServices] IImportNotificationsAggregationService importService, [FromServices] IMovementsAggregationService movementsService, + [AsParameters] DateRange dateRange, [FromQuery] string[] chartsToRender, [FromQuery(Name = "chedType")] ImportNotificationTypeEnum[] chedTypes, [FromQuery(Name = "coo")] string? countryOfOrigin, - // DateRange dateRange, - // [FromQuery] DateRange? dateRange = DateRange.Default, - [ModelBinder(typeof(DateRangeBinder))] DateRange dateRange, - // DateRange dateRange, - // [FromQuery(Name = "dateFrom")] DateTime? dateFrom, - // [FromQuery(Name = "dateTo")] DateTime? dateTo, [FromQuery(Name = "finalisedOnly")] bool finalisedOnly = true) { var logger = ApplicationLogging.CreateLogger("AnalyticsEndpoints"); @@ -133,7 +126,7 @@ private static async Task GetDashboard( var result = await AnalyticsDashboards .GetCharts(logger, importService, movementsService, chartsToRender, - chedTypes, countryOfOrigin, DateRange.Default(), finalisedOnly); + chedTypes, countryOfOrigin, dateRange, finalisedOnly); var options = new JsonSerializerOptions From 38ac59dbfc017bd20bf0ad4e130db112d061c0cb Mon Sep 17 00:00:00 2001 From: Craig Edmunds Date: Tue, 28 Jan 2025 10:31:23 +0000 Subject: [PATCH 4/4] Fixing sonarqube --- Btms.Backend/Config/AnalyticsDashboards.cs | 81 +++------------------- Btms.Backend/Program.cs | 12 ---- 2 files changed, 9 insertions(+), 84 deletions(-) diff --git a/Btms.Backend/Config/AnalyticsDashboards.cs b/Btms.Backend/Config/AnalyticsDashboards.cs index f1973a41..8f83385b 100644 --- a/Btms.Backend/Config/AnalyticsDashboards.cs +++ b/Btms.Backend/Config/AnalyticsDashboards.cs @@ -10,84 +10,21 @@ namespace Btms.Backend.Config; - -[ModelBinder(BinderType = typeof(DateRangeBinder))] public class DateRange { [FromQuery(Name = "dateFrom")] public DateTime? From { get; set; } [FromQuery(Name = "dateTo")] public DateTime? To { get; set; } - public static DateRange Default() - { - return new DateRange() { From = DateTime.Now, To = DateTime.Now }; - } - public static bool TryParse(string query, out DateRange dateRange) - { - dateRange = new DateRange() { From = DateTime.Now, To = DateTime.Now }; - return true; - } -} - -public class DateRangeBinderProvider : IModelBinderProvider -{ - public DateRangeBinderProvider() - { - Console.WriteLine("Testing12"); - } - public IModelBinder GetBinder(ModelBinderProviderContext context) - { - if (context == null) - { - throw new ArgumentNullException(nameof(context)); - } - - if (context.Metadata.ModelType == typeof(DateRange)) - { - return new BinderTypeModelBinder(typeof(DateRangeBinder)); - } - - return new BinderTypeModelBinder(context.Metadata.ModelType); - // ; - // return null!; //new BinderTypeModelBinder(); - } -} - -public class DateRangeBinder: IModelBinder -{ - public DateRangeBinder() - { - Console.WriteLine("Testing12"); - } - public Task BindModelAsync(ModelBindingContext bindingContext) - { - if (bindingContext == null) - { - throw new ArgumentNullException(nameof(bindingContext)); - } - - var dateFrom = bindingContext.ValueProvider.GetValue("dateFrom").FirstValue; - var dateTo = bindingContext.ValueProvider.GetValue("dateTo").FirstValue; - - var dateRange = new DateRange() - { - From = DateTime.Now, To = DateTime.Now - }; - // var name = bindingContext.ValueProvider.GetValue("name").FirstValue; - // var beginName = bindingContext.ValueProvider.GetValue("name:contains").FirstValue; - // var exactName = bindingContext.ValueProvider.GetValue("name:exact").FirstValue; - // if (name.ToLower().Contains("tom")) { - // model1.name = name; - // } - // if (beginName.ToLower().StartsWith("tom")) { - // model1.beginName = beginName; - // } - // if (exactName.Contains("Tom")) { - // model1.exactName = exactName; - // } - bindingContext.Result = ModelBindingResult.Success(dateRange); - return Task.CompletedTask; - } + // public static DateRange Default() + // { + // return new DateRange() { From = DateTime.Now, To = DateTime.Now }; + // } + // public static bool TryParse(string query, out DateRange dateRange) + // { + // dateRange = new DateRange() { From = DateTime.Now, To = DateTime.Now }; + // return true; + // } } public static class AnalyticsDashboards diff --git a/Btms.Backend/Program.cs b/Btms.Backend/Program.cs index 76ce1eda..d76950eb 100644 --- a/Btms.Backend/Program.cs +++ b/Btms.Backend/Program.cs @@ -26,7 +26,6 @@ using JsonApiDotNetCore.Serialization.Response; using Microsoft.AspNetCore.Diagnostics.HealthChecks; using Microsoft.Extensions.Options; -// using Microsoft.AspNetCore.Mvc.ModelBinding.Binders; using OpenTelemetry; using OpenTelemetry.Metrics; using OpenTelemetry.Trace; @@ -75,8 +74,6 @@ static void ConfigureWebApplication(WebApplicationBuilder builder) builder.Configuration.AddEnvironmentVariables(); builder.Services.AddOutputCache(options => { - // options.AddBasePolicy(builder => - // builder.Expire(TimeSpan.FromMinutes(10))); options.AddPolicy("Expire10Min", builder => builder.Expire(TimeSpan.FromMinutes(10))); } @@ -102,15 +99,6 @@ static void ConfigureWebApplication(WebApplicationBuilder builder) builder.Services.AddBusinessServices(builder.Configuration); builder.Services.AddConsumers(builder.Configuration); - - // var provider = new SimpleTypeModelBinderProvider( - // typeof(DateRange), new AnalyticsEndpoints.DateRangeBinder()); - // - // builder.Services.Insert(typeof(ModelBinderProvider), 0, provider); - builder.Services.AddControllers(options => - { - options.ModelBinderProviders.Insert(0, new DateRangeBinderProvider()); - }); ConfigureEndpoints(builder);