Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

perf(adresmatch): various performance improvements #1119

Merged
merged 2 commits into from
Nov 17, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ namespace AddressRegistry.Api.Oslo.AddressMatch
{
using System.Collections.Generic;

internal class DefaultWarningLogger : IWarningLogger
internal sealed class DefaultWarningLogger : IWarningLogger
{
public List<string> Warnings { get; } = new List<string>();

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace AddressRegistry.Api.Oslo.AddressMatch
{
public class HouseNumberWithSubaddress
public sealed class HouseNumberWithSubaddress
{
public string HouseNumber { get; }
public string? BoxNumber { get; }
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace AddressRegistry.Api.Oslo.AddressMatch
{
public class ManualAddressMatchConfig
public sealed class ManualAddressMatchConfig
{
public double FuzzyMatchThreshold { get; }

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ namespace AddressRegistry.Api.Oslo.AddressMatch
/// </summary>
/// <typeparam name="T"></typeparam>
/// <typeparam name="TProp"></typeparam>
public class PropertyEqualityComparer<T, TProp> : IEqualityComparer<T>
public sealed class PropertyEqualityComparer<T, TProp> : IEqualityComparer<T>
{
private readonly Func<T, TProp> _propertyGetter;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ public AddressMapper(ResponseOptions responseOptions, ILatestQueries latestQueri

public AddressMatchScoreableItemV2 Map(AddressDetailItemV2 source)
{
var streetName = _latestQueries.GetAllLatestStreetNames().Single(x => x.PersistentLocalId == source.StreetNamePersistentLocalId);
var municipality = _latestQueries.GetAllLatestMunicipalities().Single(x => x.NisCode == streetName.NisCode);
var streetName = _latestQueries.GetAllLatestStreetNamesByPersistentLocalId()[source.StreetNamePersistentLocalId];
var municipality = _latestQueries.GetAllLatestMunicipalities()[streetName.NisCode];
var defaultStreetName = Address.AddressMapper.GetDefaultStreetNameName(streetName, municipality.PrimaryLanguage);
var homonym = Address.AddressMapper.GetDefaultHomonymAddition(streetName, municipality.PrimaryLanguage);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ namespace AddressRegistry.Api.Oslo.AddressMatch.V2.Matching
using AddressRegistry.Projections.Legacy.AddressDetailV2;
using Convertors;

public class AddressMatchBuilder : IEnumerable<AddressMatchBuilder.MunicipalityWrapper>, IProvidesRepresentationsForScoring
public sealed class AddressMatchBuilder : IEnumerable<AddressMatchBuilder.MunicipalityWrapper>, IProvidesRepresentationsForScoring
{
public class StreetNameWrapper : IEnumerable<AddressDetailItemV2>
public sealed class StreetNameWrapper : IEnumerable<AddressDetailItemV2>
{
private IList<AddressDetailItemV2> _addresses;

Expand All @@ -36,7 +36,7 @@ IEnumerator IEnumerable.GetEnumerator()
=> GetEnumerator();
}

public class MunicipalityWrapper : IEnumerable<StreetNameWrapper>
public sealed class MunicipalityWrapper : IEnumerable<StreetNameWrapper>
{
private IList<StreetNameWrapper> _streetNames;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ namespace AddressRegistry.Api.Oslo.AddressMatch.V2.Matching
/// Implements an algorithm for matching AdresMatchQueryComponents to Gemeentes, Straatnamen or Adressen and scoring the results.
/// </summary>
/// <typeparam name="TResult"></typeparam>
public class AddressMatchMatchingAlgorithm<TResult> : MatchingAlgorithm<AddressMatchBuilder, TResult>
public sealed class AddressMatchMatchingAlgorithm<TResult> : MatchingAlgorithm<AddressMatchBuilder, TResult>
where TResult : class, IScoreable
{
public AddressMatchMatchingAlgorithm(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace AddressRegistry.Api.Oslo.AddressMatch.V2.Matching
{
public class AddressMatchQueryComponents
public sealed class AddressMatchQueryComponents
{
public string MunicipalityName { get; set; }
public string NisCode { get; set; }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ namespace AddressRegistry.Api.Oslo.AddressMatch.V2.Matching
using Be.Vlaanderen.Basisregisters.GrAr.Legacy.SpatialTools;
using Responses;

public class AddressMatchScoreableItemV2 : IScoreable
public sealed class AddressMatchScoreableItemV2 : IScoreable
{
public int AddressPersistentLocalId { get; set; }
public AdresIdentificator Identificator { get; set; }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ namespace AddressRegistry.Api.Oslo.AddressMatch.V2.Matching
using System.Linq;
using AddressRegistry.Projections.Legacy.AddressDetailV2;

internal class AddressMatcher<TResult> : ScoreableObjectMatcherBase<AddressMatchBuilder, TResult>
internal sealed class AddressMatcher<TResult> : ScoreableObjectMatcherBase<AddressMatchBuilder, TResult>
where TResult : class, IScoreable
{
private readonly ILatestQueries _latestQueries;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,16 @@ namespace AddressRegistry.Api.Oslo.AddressMatch.V2.Matching

public interface ILatestQueries
{
IEnumerable<MunicipalityLatestItem> GetAllLatestMunicipalities();
IDictionary<string, MunicipalityLatestItem> GetAllLatestMunicipalities();
IEnumerable<StreetNameLatestItem> GetAllLatestStreetNames();
IDictionary<int, StreetNameLatestItem> GetAllLatestStreetNamesByPersistentLocalId();
IEnumerable<StreetNameLatestItem> GetLatestStreetNamesBy(params string[] municipalityNames);
StreetNameLatestItem FindLatestStreetNameById(int streetNamePersistentLocalId);
StreetNameLatestItem? FindLatestStreetNameById(int streetNamePersistentLocalId);
IEnumerable<AddressDetailItemV2> GetLatestAddressesBy(int streetNamePersistentLocalId, string? houseNumber, string? boxNumber);
IEnumerable<PostalInfoLatestItem> GetAllPostalInfo();
}

public class CachedLatestQueriesDecorator : CachedService, ILatestQueries
public sealed class CachedLatestQueriesDecorator : CachedService, ILatestQueries
{
private readonly AddressMatchContextV2 _context;
private static readonly TimeSpan AllStreetNamesCacheDuration = TimeSpan.FromDays(1);
Expand All @@ -36,18 +37,21 @@ public CachedLatestQueriesDecorator(
AddressMatchContextV2 context)
: base(memoryCache) => _context = context;

public IEnumerable<MunicipalityLatestItem> GetAllLatestMunicipalities() =>
public IDictionary<string, MunicipalityLatestItem> GetAllLatestMunicipalities() =>
GetOrAdd(
AllMunicipalitiesCacheKey,
() => _context.MunicipalityLatestItems.ToList(),
() => _context.MunicipalityLatestItems.ToDictionary(x => x.NisCode, x => x),
AllMunicipalitiesCacheDuration);

public IEnumerable<StreetNameLatestItem> GetAllLatestStreetNames() =>
public IDictionary<int, StreetNameLatestItem> GetAllLatestStreetNamesByPersistentLocalId() =>
GetOrAdd(
AllStreetNamesCacheKey,
() => ConvertToCachedModel(GetLatestStreetNameItems().ToList()),
() => new StreetNameCache(GetLatestStreetNameItems().ToList()),
AllStreetNamesCacheDuration)
.SelectMany(kvp => kvp.Value);
.ByPersistentLocalId;

public IEnumerable<StreetNameLatestItem> GetAllLatestStreetNames() =>
GetAllLatestStreetNamesByPersistentLocalId().Values;

public IEnumerable<StreetNameLatestItem> GetLatestStreetNamesBy(params string[] municipalityNames)
{
Expand All @@ -57,27 +61,26 @@ public IEnumerable<StreetNameLatestItem> GetLatestStreetNamesBy(params string[]

var nisCodes = GetAllLatestMunicipalities()
.Where(x =>
lowerMunicipalityNames.Contains(x.NameDutchSearch) || // English/German probably not needed as AddressMatch is relevant to flanders
lowerMunicipalityNames.Contains(x.NameFrenchSearch))
.Select(x => x.NisCode);
lowerMunicipalityNames.Contains(x.Value.NameDutchSearch) || // English/German probably not needed as AddressMatch is relevant to flanders
lowerMunicipalityNames.Contains(x.Value.NameFrenchSearch))
.Select(x => x.Key);

return GetOrAddLatestStreetNames(
streetNames => nisCodes
.SelectMany(g => streetNames.ContainsKey(g)
? streetNames[g]
: new StreetNameLatestItem[] { })
.SelectMany(g => streetNames.ByMunicipalityNisCode.TryGetValue(g, out var value)
? value
: Array.Empty<StreetNameLatestItem>())
.AsQueryable(),
() => GetLatestStreetNameItems()
.Where(x => nisCodes.Contains(x.NisCode))
.ToList()
.AsEnumerable());
}

public StreetNameLatestItem FindLatestStreetNameById(int streetNamePersistentLocalId) =>
public StreetNameLatestItem? FindLatestStreetNameById(int streetNamePersistentLocalId) =>
GetOrAddLatestStreetNames(
streetNames => streetNames
.SelectMany(kvp => kvp.Value)
.Single(s => s.PersistentLocalId == streetNamePersistentLocalId),
.ByPersistentLocalId[streetNamePersistentLocalId],
() => GetLatestStreetNameItems().FirstOrDefault(
x => x.PersistentLocalId == streetNamePersistentLocalId));

Expand Down Expand Up @@ -112,18 +115,13 @@ private IQueryable<StreetNameLatestItem> GetLatestStreetNameItems()
.Where(x => !x.IsRemoved);

private T GetOrAddLatestStreetNames<T>(
Func<Dictionary<string, IEnumerable<StreetNameLatestItem>>, T> ifCacheHit,
Func<StreetNameCache, T> ifCacheHit,
Func<T> ifCacheNotHit)
=> GetOrAdd(
AllStreetNamesCacheKey,
() => ConvertToCachedModel(GetLatestStreetNameItems().ToList()),
() => new StreetNameCache(GetLatestStreetNameItems().ToList()),
AllStreetNamesCacheDuration,
ifCacheHit,
ifCacheNotHit);

private static Dictionary<string, IEnumerable<StreetNameLatestItem>> ConvertToCachedModel(IEnumerable<StreetNameLatestItem> query)
=> query
.GroupBy(s => s.NisCode)
.ToDictionary(s => s.Key, s => s.AsEnumerable());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ namespace AddressRegistry.Api.Oslo.AddressMatch.V2.Matching
using AddressRegistry.Consumer.Read.Municipality.Projections;
using Be.Vlaanderen.Basisregisters.GrAr.Common;

internal class MunicipalityNameComparer : EqualityComparer<AddressMatchBuilder.MunicipalityWrapper>
internal sealed class MunicipalityNameComparer : EqualityComparer<AddressMatchBuilder.MunicipalityWrapper>
{
public override bool Equals(
AddressMatchBuilder.MunicipalityWrapper x,
Expand All @@ -19,7 +19,7 @@ public override int GetHashCode(AddressMatchBuilder.MunicipalityWrapper obj)
: obj.Name.GetHashCode();
}

internal class MunicipalityMatcher<TResult> : ScoreableObjectMatcherBase<AddressMatchBuilder, TResult>
internal sealed class MunicipalityMatcher<TResult> : ScoreableObjectMatcherBase<AddressMatchBuilder, TResult>
where TResult : class, IScoreable
{
private readonly ManualAddressMatchConfig _config;
Expand All @@ -43,6 +43,7 @@ public MunicipalityMatcher(
{
var municipalities = _latestQueries
.GetAllLatestMunicipalities()
.Values
.ToList();

var municipalitiesByName = FilterByName(municipalities, results);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
namespace AddressRegistry.Api.Oslo.AddressMatch.V2.Matching
{
using System.Collections.Generic;
using System.Linq;
using Consumer.Read.StreetName.Projections;

public sealed class StreetNameCache
{
public StreetNameCache(IList<StreetNameLatestItem> streetNameLatestItems)
{
ByPersistentLocalId = streetNameLatestItems.ToDictionary(x => x.PersistentLocalId);
ByMunicipalityNisCode = streetNameLatestItems
.GroupBy(x => x.NisCode)
.ToDictionary(x => x.Key!, x => x.AsEnumerable());
}

public Dictionary<int,StreetNameLatestItem> ByPersistentLocalId { get; private set; }
public Dictionary<string, IEnumerable<StreetNameLatestItem>> ByMunicipalityNisCode { get; private set; }
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ namespace AddressRegistry.Api.Oslo.AddressMatch.V2.Matching
using AddressRegistry.Consumer.Read.StreetName.Projections;
using Be.Vlaanderen.Basisregisters.GrAr.Common;

internal class StreetNameMatcher<TResult> : ScoreableObjectMatcherBase<AddressMatchBuilder, TResult>
internal sealed class StreetNameMatcher<TResult> : ScoreableObjectMatcherBase<AddressMatchBuilder, TResult>
where TResult : class, IScoreable
{
private readonly ILatestQueries _latestQueries;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ namespace AddressRegistry.Api.Oslo.AddressMatch.V2
using Matching;
using Responses;

internal class MunicipalityMapper : IMapper<MunicipalityLatestItem, AddressMatchScoreableItemV2>
internal sealed class MunicipalityMapper : IMapper<MunicipalityLatestItem, AddressMatchScoreableItemV2>
{
private readonly ResponseOptions _responseOptions;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
namespace AddressRegistry.Api.Oslo.AddressMatch.V2
{
using System.Linq;
using AddressRegistry.Consumer.Read.StreetName.Projections;
using Be.Vlaanderen.Basisregisters.GrAr.Legacy;
using Be.Vlaanderen.Basisregisters.GrAr.Legacy.Gemeente;
Expand All @@ -22,7 +21,7 @@ public StreetNameMapper(ResponseOptions responseOptions, ILatestQueries latestQu

public AddressMatchScoreableItemV2 Map(StreetNameLatestItem source)
{
var municipality = _latestQueries.GetAllLatestMunicipalities().Single(x => x.NisCode == source.NisCode);
var municipality = _latestQueries.GetAllLatestMunicipalities()[source.NisCode];
var name = Address.AddressMapper.GetDefaultStreetNameName(source, municipality.PrimaryLanguage);
var homonym = Address.AddressMapper.GetDefaultHomonymAddition(source, municipality.PrimaryLanguage);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ namespace AddressRegistry.Api.Oslo.AddressMatch
using System.Collections.Generic;
using System.Runtime.Serialization;

internal class ValidationMessageWarningLogger : IWarningLogger
internal sealed class ValidationMessageWarningLogger : IWarningLogger
{
public List<ValidationOsloMessage> Warnings { get; }

Expand All @@ -19,7 +19,7 @@ public ValidationMessageWarningLogger()
/// contains a warning message in dutch and english
/// </summary>
[DataContract(Name = "Warning", Namespace = "")]
public class ValidationOsloMessage
public sealed class ValidationOsloMessage
{
/// <summary>
/// Code van de warning.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ namespace AddressRegistry.Api.Legacy.Tests.AddressMatch
using Framework.Generate;
using Framework.Mocking;
using Legacy.AddressMatch.Requests;
using Microsoft.AspNetCore.Mvc;
using Projections.Legacy.AddressDetail;
using Xunit;
using Xunit.Abstractions;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ public async Task CanFindGemeenteByGemeentenaam_FuzzyMatch(string requestedGemee
NameDutchSearch = existingGemeentenaam.RemoveDiacritics(),
OfficialLanguages = new List<string> { "Dutch" }
}
});
}.ToDictionary(x => x.NisCode));

//Act
var response = await _handler.Handle(request, CancellationToken.None);
Expand Down Expand Up @@ -134,7 +134,7 @@ public async Task CanFindGemeenteByGemeentenaam_NoMatch()
NameDutchSearch = "Springfield".RemoveDiacritics(),
OfficialLanguages = new List<string> { "Dutch" }
}
});
}.ToDictionary(x => x.NisCode));

//Act
var response = await _handler.Handle(request, CancellationToken.None);
Expand Down Expand Up @@ -421,7 +421,7 @@ private MunicipalityLatestItem MockGetAllLatestMunicipalities(string nisCode, st
.Returns(new[]
{
municipalityLatestItem
});
}.ToDictionary(x => x.NisCode));

return municipalityLatestItem;
}
Expand All @@ -437,7 +437,7 @@ private void MockGetAllLatestMunicipalities(IEnumerable<string> nisCodes, string
NameDutch = municipalityName,
NameDutchSearch = municipalityName.RemoveDiacritics(),
OfficialLanguages = new List<string> { "Dutch" }
})
}).ToDictionary(x => x.NisCode)
);
}

Expand All @@ -463,6 +463,7 @@ private void MockGetAllPostalInfo(string nisCode, string postalCode, string post

private StreetNameLatestItem MockStreetNames(string streetName, int streetNamePersistentLocalId, string nisCode, string municipalityName)
{

var streetNameLatestItem = new StreetNameLatestItem(streetNamePersistentLocalId, nisCode)
{
NameDutch = streetName,
Expand All @@ -472,6 +473,9 @@ private StreetNameLatestItem MockStreetNames(string streetName, int streetNamePe
_latestQueries
.Setup(x => x.GetAllLatestStreetNames())
.Returns(new[] { streetNameLatestItem });
_latestQueries
.Setup(x => x.GetAllLatestStreetNamesByPersistentLocalId())
.Returns(new[] { streetNameLatestItem }.ToDictionary(x => x.PersistentLocalId));
_latestQueries
.Setup(x => x.GetLatestStreetNamesBy(municipalityName))
.Returns(new[] { streetNameLatestItem });
Expand Down
Loading