Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CDMS-155 allows finalisedOnly to be passed to analytics endpoints to … #83

Merged
merged 4 commits into from
Jan 28, 2025
Merged

CDMS-155 allows finalisedOnly to be passed to analytics endpoints to … #83

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
32a0141
CDMS-155 allows finalisedOnly to be passed to analytics endpoints to …
craigedmunds Jan 27, 2025
01f9900
CDMS-155 attempting to fix sonarqube 'too many arguments' error
craigedmunds Jan 28, 2025
c25b858
Using AsParams
craigedmunds Jan 28, 2025
22e9856
Fixing sonarqube
craigedmunds Jan 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions Btms.Analytics.Tests/FinalisedOnlyTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
using Btms.Common.Extensions;
using FluentAssertions;
using Xunit;
using Xunit.Abstractions;

using Btms.Analytics.Tests.Fixtures;
using Btms.Analytics.Tests.Helpers;
using TestDataGenerator.Config;
using TestDataGenerator.Scenarios;
using TestDataGenerator.Scenarios.SpecificFiles;
using TestGenerator.IntegrationTesting.Backend;
using TestGenerator.IntegrationTesting.Backend.Extensions;
using TestGenerator.IntegrationTesting.Backend.Fixtures;

namespace Btms.Analytics.Tests;

public class FinalisedOnlyTests(ITestOutputHelper output) : MultipleScenarioGeneratorBaseTest(output)
{
[Theory]
[InlineData(typeof(Mrn24GBDDJER3ZFRMZAR9ScenarioGenerator), false, true)]
[InlineData(typeof(Mrn24GBDDJER3ZFRMZAR9ScenarioGenerator), true, false)]
public async Task ShouldReturnCorrectAggregation(Type generatorType, bool finalisedOnly, bool returnsResults)
{
EnsureEnvironmentInitialised(generatorType);

var result = await Client
.GetAnalyticsDashboard(["decisionsByDecisionCode"],
dateFrom: DateTime.MinValue, dateTo: DateTime.MaxValue,
finalisedOnly: finalisedOnly);

var chart = await result
.AnalyticsChartAs<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>>("decisionsByDecisionCode")!;

chart.Summary.Values.Count
.Should().Be(returnsResults ? 1 : 0);
}
}
10 changes: 5 additions & 5 deletions Btms.Analytics.Tests/ImportNotificationsByMaxVersionTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ public async Task WhenCalledLastMonth_ReturnExpectedAggregation()
{
TestOutputHelper.WriteLine("Querying for aggregated data");
var result = (await GetImportNotificationsAggregationService()
.ByMaxVersion(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow()));
.ByMaxVersion(DateTime.Today.MonthAgo(), DateTime.Today.Tomorrow(), false));

TestOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count);

Expand All @@ -31,7 +31,7 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
{
TestOutputHelper.WriteLine("Querying for aggregated data");
var result = await ImportNotificationsAggregationService
.ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour());
.ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), false);

TestOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");

Expand All @@ -43,7 +43,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati
{
TestOutputHelper.WriteLine("Querying for aggregated data");
var result = await ImportNotificationsAggregationService
.ByMaxVersion(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue);
.ByMaxVersion(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue, false);

TestOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");

Expand All @@ -55,7 +55,7 @@ public async Task WhenCalledWithChedType_ReturnsResults()
{
TestOutputHelper.WriteLine("Querying for aggregated data");
var result = await ImportNotificationsAggregationService
.ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), chedTypes: [ImportNotificationTypeEnum.Cveda]);
.ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), false, chedTypes: [ImportNotificationTypeEnum.Cveda]);

TestOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");

Expand All @@ -67,7 +67,7 @@ public async Task WhenCalledWithCountry_ReturnsResults()
{
TestOutputHelper.WriteLine("Querying for aggregated data");
var result = await ImportNotificationsAggregationService
.ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), country: "ES");
.ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), false, country: "ES");

TestOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");

Expand Down
3 changes: 1 addition & 2 deletions Btms.Analytics.Tests/MovementsByDecisionsTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,11 @@ public class MovementsByDecisionsTests(ITestOutputHelper output)
: ScenarioDatasetBaseTest(output, Datasets.FunctionalAnalyticsDecisionsDatasetName)
{
[Fact]
// [Fact(Skip = "Needs revisiting - needs more assertions, perhaps switch to individual scenario test")]
public async Task WhenCalled_ReturnExpectedAggregation()
{
TestOutputHelper.WriteLine("Querying for aggregated data");
var result = await MovementsAggregationService
.ByDecision(DateTime.MinValue, DateTime.MaxValue)!;
.ByDecision(DateTime.MinValue, DateTime.MaxValue, false)!;

TestOutputHelper.WriteLine("{0} aggregated items found", result!.Result.Count());

Expand Down
1 change: 1 addition & 0 deletions Btms.Analytics/Extensions/ImportNotificationExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ namespace Btms.Analytics.Extensions;
public static class ImportNotificationExtensions
{
public static IQueryable<ImportNotification> WhereFilteredByCreatedDateAndParams(this IQueryable<ImportNotification> source, DateTime from, DateTime to,
bool finalisedOnly = true,
ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
{
return source
Expand Down
17 changes: 4 additions & 13 deletions Btms.Analytics/Extensions/MovementExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,29 +6,20 @@ namespace Btms.Analytics.Extensions;

public static class MovementExtensions
{
public static IQueryable<Movement> WhereFilteredByCreatedDateAndParams(this IQueryable<Movement> source, DateTime from, DateTime to,
public static IQueryable<Movement> WhereFilteredByCreatedDateAndParams(this IQueryable<Movement> source, DateTime from, DateTime to,
bool finalisedOnly,
ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
{
return source
.Where(m => (m.CreatedSource >= from && m.CreatedSource < to)
&& (country == null || m.DispatchCountryCode == country)
&& (!finalisedOnly || m.Finalised.HasValue)
&& (chedTypes == null || !chedTypes!.Any() ||
!m.BtmsStatus.ChedTypes!.Any() ||
m.BtmsStatus.ChedTypes!.Any(c => chedTypes!.Contains(c))));

}
// public static IQueryable<Movement> WhereFilteredByCreatedDateAndParams(this IQueryable<Movement> source, DateTime from, DateTime to,
// ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
// {
// return source
// .Where(m => (m.CreatedSource >= from && m.CreatedSource < to)
// && (country == null || m.DispatchCountryCode == country)
// && (chedTypes == null || !chedTypes!.Any() ||
// !m.BtmsStatus.ChedTypes!.Any() ||
// m.BtmsStatus.ChedTypes!.Any(c => chedTypes!.Contains(c))));
//
// }


public class MovementWithLinkStatus
{
public required Movement Movement;
Expand Down
2 changes: 1 addition & 1 deletion Btms.Analytics/IImportNotificationsAggregationService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,6 @@ public interface IImportNotificationsAggregationService
public Task<MultiSeriesDatetimeDataset> ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
public Task<SingleSeriesDataset> ByStatus(DateTime? from = null, DateTime? to = null);
public Task<MultiSeriesDataset<ByNumericDimensionResult>> ByCommodityCount(DateTime from, DateTime to);
public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
public EntityDataset<ScenarioItem> Scenarios(DateTime? from, DateTime? to);
}
12 changes: 6 additions & 6 deletions Btms.Analytics/IMovementsAggregationService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,16 @@ public interface IMovementsAggregationService
public Task<MultiSeriesDatetimeDataset> ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
public Task<SingleSeriesDataset> ByStatus(DateTime? from = null, DateTime? to = null);
public Task<MultiSeriesDataset<ByNumericDimensionResult>> ByItemCount(DateTime from, DateTime to);
public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>> ByDecision(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>> ByDecision(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
// public Task<TabularDataset<ByNameDimensionResult>> ByDecisionAndLinkStatus(DateTime from, DateTime to);
public Task<MultiSeriesDataset<ByNumericDimensionResult>> ByUniqueDocumentReferenceCount(DateTime from, DateTime to);
public Task<SingleSeriesDataset> UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to);
// public Task<MultiSeriesDataset> ByCheck(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
public Task<EntityDataset<AuditHistory>?> GetHistory(string movementId);
public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
public Task<SingleSeriesDataset> ByMaxDecisionNumber(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
public Task<List<ExceptionResult>> GetExceptions(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
public Task<SingleSeriesDataset> ExceptionSummary(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
public Task<SingleSeriesDataset> ByMaxDecisionNumber(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
public Task<List<ExceptionResult>> GetExceptions(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
public Task<SingleSeriesDataset> ExceptionSummary(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>> BySegment(DateTime from,
DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
}
7 changes: 5 additions & 2 deletions Btms.Analytics/ImportNotificationsAggregationService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,11 +161,14 @@ public Task<MultiSeriesDataset<ByNumericDimensionResult>> ByCommodityCount(DateT
});
}

public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
{
// NB : At the moment this doesn't filter on finalisedOnly as thats not stored anywhere on the notification
// we'd need to denormalise the field, perhaps onto the relationship, to allow this filtering.

var data = context
.Notifications
.WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country)
.WhereFilteredByCreatedDateAndParams(from, to, finalisedOnly, chedTypes, country)
.GroupBy(n => new { MaxVersion =
n.AuditEntries.Where(a => a.CreatedBy == CreatedBySystem.Ipaffs).Max(a => a.Version )
})
Expand Down
4 changes: 2 additions & 2 deletions Btms.Analytics/MovementExceptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,14 @@ public class MovementExceptions(IMongoDbContext context, ILogger logger)
//Returns a summary of the exceptions or a list
// Means we can share the same anonymous / query code without needing to create loads
// of classes
public (SingleSeriesDataset summary, List<ExceptionResult>) GetAllExceptions(DateTime from, DateTime to, bool summary, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
public (SingleSeriesDataset summary, List<ExceptionResult>) GetAllExceptions(DateTime from, DateTime to, bool finalisedOnly, bool summary, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
{
var exceptionsSummary = new SingleSeriesDataset();
var exceptionsResult = new List<ExceptionResult>();

var simplifiedMovementView = context
.Movements
.WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country)
.WhereFilteredByCreatedDateAndParams(from, to, finalisedOnly, chedTypes, country)
.Where(m => m.BtmsStatus.LinkStatus != LinkStatusEnum.AllLinked)
.Select(m => new
{
Expand Down
32 changes: 17 additions & 15 deletions Btms.Analytics/MovementsAggregationService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,11 +199,11 @@ public Task<SingleSeriesDataset> UniqueDocumentReferenceByMovementCount(DateTime
return new EntityDataset<AuditHistory>(entries);
}

public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
{
var data = context
.Movements
.WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country)
.WhereFilteredByCreatedDateAndParams(from, to, finalisedOnly, chedTypes, country)
.GroupBy(n => new { MaxVersion =
n.ClearanceRequests.Max(a => a.Header!.EntryVersionNumber )
})
Expand All @@ -216,11 +216,11 @@ public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, Import
});
}

public Task<SingleSeriesDataset> ByMaxDecisionNumber(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
public Task<SingleSeriesDataset> ByMaxDecisionNumber(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
{
var data = context
.Movements
.WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country)
.WhereFilteredByCreatedDateAndParams(from, to, finalisedOnly, chedTypes, country)
.GroupBy(n => new { MaxVersion =
n.Decisions.Max(a => a.Header!.DecisionNumber )
})
Expand All @@ -233,20 +233,20 @@ public Task<SingleSeriesDataset> ByMaxDecisionNumber(DateTime from, DateTime to,
});
}

public Task<List<ExceptionResult>> GetExceptions(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
public Task<List<ExceptionResult>> GetExceptions(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
{
var movementExceptions = new MovementExceptions(context, logger);
var (_, result) = movementExceptions
.GetAllExceptions(from, to, false, chedTypes, country);
.GetAllExceptions(from, to, finalisedOnly, false, chedTypes, country);

return Task.FromResult(result);
}

public Task<SingleSeriesDataset> ExceptionSummary(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
public Task<SingleSeriesDataset> ExceptionSummary(DateTime from, DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
{
var movementExceptions = new MovementExceptions(context, logger);
var (summary, _) = movementExceptions
.GetAllExceptions(from, to, true, chedTypes, country);
.GetAllExceptions(from, to, finalisedOnly, true, chedTypes, country);

return Task.FromResult(summary);
}
Expand Down Expand Up @@ -283,15 +283,16 @@ private Task<MultiSeriesDatetimeDataset> Aggregate(DateTime[] dateRange, Func<Bs
/// </summary>
/// <param name="from"></param>
/// <param name="to"></param>
/// <param name="finalisedOnly"></param>
/// <param name="chedTypes"></param>
/// <param name="country"></param>
/// <returns></returns>
public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>> ByDecision(DateTime from,
DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
{
var mongoQuery = context
.Movements
.WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country)
.WhereFilteredByCreatedDateAndParams(from, to, finalisedOnly, chedTypes, country)
.SelectMany(d => d
.AlvsDecisionStatus.Context.DecisionComparison!.Checks
.Select(c => new
Expand Down Expand Up @@ -367,11 +368,11 @@ public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>>
}

public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>> BySegment(DateTime from,
DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
DateTime to, bool finalisedOnly, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
{
var mongoQuery = context
.Movements
.WhereFilteredByCreatedDateAndParams(from, to, chedTypes, country)
.WhereFilteredByCreatedDateAndParams(from, to, finalisedOnly, chedTypes, country)
.Select(m => new
{
DecisionStatus = m.AlvsDecisionStatus.Context.DecisionComparison == null ?
Expand All @@ -383,7 +384,8 @@ public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>>
})
.GroupBy(d => new
{
Segment = d.BtmsStatus.Segment ?? MovementSegmentEnum.None,
d.BtmsStatus.Segment,
// Segment = d.BtmsStatus.Segment ?? MovementSegmentEnum.None,
d.BtmsStatus.LinkStatus,
d.BtmsStatus.Status,
d.DecisionStatus,
Expand All @@ -409,7 +411,7 @@ public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>>
.OrderBy(s => s.Key)
// .Where(g => g)
.ToDictionary(
g => enumLookup.GetValue(g.Key),
g => enumLookup.GetValue(g.Key ?? MovementSegmentEnum.None),
g => g.Sum
);

Expand All @@ -426,7 +428,7 @@ public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>>
Fields = new Dictionary<string, string>()
{
// { "Classification", enumLookup.GetValue(a.Key!.Segment) },
{ "Classification", enumLookup.GetValue(a.Key!.Segment!) },
{ "Classification", enumLookup.GetValue(a.Key.Segment ?? MovementSegmentEnum.None) },
// { "CheckCode", a.Key.CheckCode! },
// { "AlvsDecisionCode", a.Key.AlvsDecisionCode! },
// { "BtmsDecisionCode", a.Key.BtmsDecisionCode! }
Expand Down
Loading
Loading