diff --git a/AssociationRegistry.sln.DotSettings b/AssociationRegistry.sln.DotSettings index 2220c66c5..ce86b0d8c 100644 --- a/AssociationRegistry.sln.DotSettings +++ b/AssociationRegistry.sln.DotSettings @@ -238,6 +238,7 @@ True True True + True True True True diff --git a/src/AssociationRegistry.Admin.Api/DuplicateDetection/SearchDuplicateVerenigingDetectionService.cs b/src/AssociationRegistry.Admin.Api/DuplicateDetection/SearchDuplicateVerenigingDetectionService.cs index c31a1d32d..d3dad2529 100644 --- a/src/AssociationRegistry.Admin.Api/DuplicateDetection/SearchDuplicateVerenigingDetectionService.cs +++ b/src/AssociationRegistry.Admin.Api/DuplicateDetection/SearchDuplicateVerenigingDetectionService.cs @@ -1,23 +1,23 @@ namespace AssociationRegistry.Admin.Api.DuplicateDetection; using DuplicateVerenigingDetection; -using Marten; -using Schema.Constants; -using Schema.Detail; +using Nest; +using Schema.Search; using System; using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; +using System.Linq.Expressions; using System.Threading.Tasks; using Vereniging; public class SearchDuplicateVerenigingDetectionService : IDuplicateVerenigingDetectionService { - private readonly IQuerySession _session; + private readonly IElasticClient _client; - public SearchDuplicateVerenigingDetectionService(IQuerySession session) + public SearchDuplicateVerenigingDetectionService(IElasticClient client) { - _session = session; + _client = client; } public async Task> GetDuplicates(VerenigingsNaam naam, Locatie[] locaties) @@ -26,39 +26,87 @@ public async Task> GetDuplicates(Vereni var postcodes = locatiesMetAdres.Select(l => l.Adres!.Postcode).ToArray(); var gemeentes = locatiesMetAdres.Select(l => l.Adres!.Gemeente).ToArray(); - return (await _session.Query() - .Where( - document => - document.Status.Equals(VerenigingStatus.Actief) && - document.Naam.Equals(naam, StringComparison.InvariantCultureIgnoreCase) && - document.Locaties.Any( - locatie => - locatie.Adres != null && ( - locatie.Adres.Postcode.IsOneOf(postcodes) || - locatie.Adres.Gemeente.IsOneOf(gemeentes)) - ) - ) - .ToListAsync()) - .Select(ToDuplicateVereniging) - .ToArray(); + var searchResponse = + await _client + .SearchAsync( + s => s.Query( + q => q.Bool( + b => b.Must(must => must.Match(m => FuzzyMatchOpNaam(m, f => f.Naam, naam))) + .Filter(f => f.Bool( + fb => fb.Should(MatchGemeente(gemeentes), + MatchPostcode(postcodes)) + .MinimumShouldMatch(1)))))); + + return searchResponse.Documents.Select(ToDuplicateVereniging) + .ToArray(); + } + + private static Func, QueryContainer> MatchPostcode(string[] postcodes) + { + return postalCodesQuery => postalCodesQuery + .Nested(n => n + .Path(p => p.Locaties) + .Query(nq => nq + .Terms(t => t + .Field(f => f.Locaties + .First() + .Postcode) + .Terms(postcodes) + ) + ) + ); + } + + private static Func, QueryContainer> MatchGemeente(string[] gemeentes) + { + return gemeentesQuery => gemeentesQuery + .Nested(n => n + .Path(p => p.Locaties) + .Query(nq => nq + .Match(m => + FuzzyMatchOpNaam(m, + f => f.Locaties + .First() + .Gemeente, string.Join( + separator: " ", + gemeentes)) + ) + ) + ); + } + + private static MatchQueryDescriptor FuzzyMatchOpNaam( + MatchQueryDescriptor m, + Expression> path, + string query) + { + return m + .Field(path) + .Query(query) + .Analyzer(DuplicateDetectionDocumentMapping + .DuplicateAnalyzer) + .Fuzziness(Fuzziness.Auto) // Assumes this analyzer applies lowercase and asciifolding + .MinimumShouldMatch("90%"); } - private static DuplicaatVereniging ToDuplicateVereniging(BeheerVerenigingDetailDocument document) + private static DuplicaatVereniging ToDuplicateVereniging(DuplicateDetectionDocument document) => new( document.VCode, - new DuplicaatVereniging.VerenigingsType(document.Type.Code, document.Type.Beschrijving), + new DuplicaatVereniging.VerenigingsType(document.VerenigingsTypeCode, + Verenigingstype.Parse(document.VerenigingsTypeCode).Beschrijving), document.Naam, - document.KorteNaam ?? string.Empty, - document.HoofdactiviteitenVerenigingsloket - .Select(h => new DuplicaatVereniging.HoofdactiviteitVerenigingsloket(h.Code, h.Beschrijving)).ToImmutableArray(), + document.KorteNaam, + document.HoofdactiviteitVerenigingsloket + .Select(h => new DuplicaatVereniging.HoofdactiviteitVerenigingsloket( + h, HoofdactiviteitVerenigingsloket.Create(h).Beschrijving)).ToImmutableArray(), document.Locaties.Select(ToLocatie).ToImmutableArray()); - private static DuplicaatVereniging.Locatie ToLocatie(BeheerVerenigingDetailDocument.Locatie loc) + private static DuplicaatVereniging.Locatie ToLocatie(DuplicateDetectionDocument.Locatie loc) => new( loc.Locatietype, loc.IsPrimair, loc.Adresvoorstelling, loc.Naam, - loc.Adres?.Postcode ?? string.Empty, - loc.Adres?.Gemeente ?? string.Empty); + loc.Postcode ?? string.Empty, + loc.Gemeente ?? string.Empty); } diff --git a/src/AssociationRegistry.Admin.Api/Infrastructure/ConfigurationBindings/ElasticSearchOptionsSection.cs b/src/AssociationRegistry.Admin.Api/Infrastructure/ConfigurationBindings/ElasticSearchOptionsSection.cs index 13ff3b81d..b84ec8c2b 100644 --- a/src/AssociationRegistry.Admin.Api/Infrastructure/ConfigurationBindings/ElasticSearchOptionsSection.cs +++ b/src/AssociationRegistry.Admin.Api/Infrastructure/ConfigurationBindings/ElasticSearchOptionsSection.cs @@ -7,9 +7,12 @@ public class ElasticSearchOptionsSection public string? Username { get; set; } public string? Password { get; set; } public IndicesOptionsSection? Indices { get; set; } + public bool EnableDevelopmentLogs { get; set; } public class IndicesOptionsSection { public string? Verenigingen { get; set; } + public string? DuplicateDetection { get; set; } + } } diff --git a/src/AssociationRegistry.Admin.Api/Infrastructure/Extensions/ElasticSearchExtensions.cs b/src/AssociationRegistry.Admin.Api/Infrastructure/Extensions/ElasticSearchExtensions.cs index 496ca4182..da2b79b90 100644 --- a/src/AssociationRegistry.Admin.Api/Infrastructure/Extensions/ElasticSearchExtensions.cs +++ b/src/AssociationRegistry.Admin.Api/Infrastructure/Extensions/ElasticSearchExtensions.cs @@ -1,10 +1,12 @@ namespace AssociationRegistry.Admin.Api.Infrastructure.Extensions; -using System; using ConfigurationBindings; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Nest; using Schema.Search; +using System; +using System.Text; public static class ElasticSearchExtensions { @@ -12,31 +14,57 @@ public static IServiceCollection AddElasticSearch( this IServiceCollection services, ElasticSearchOptionsSection elasticSearchOptions) { - var elasticClient = CreateElasticClient(elasticSearchOptions); + var elasticClient = (IServiceProvider serviceProvider) + => CreateElasticClient(elasticSearchOptions, serviceProvider.GetRequiredService>()); - services.AddSingleton(_ => elasticClient); - services.AddSingleton(_ => elasticClient); + services.AddSingleton(sp => elasticClient(sp)); + services.AddSingleton(serviceProvider => serviceProvider.GetRequiredService()); return services; } - private static ElasticClient CreateElasticClient(ElasticSearchOptionsSection elasticSearchOptions) + private static ElasticClient CreateElasticClient(ElasticSearchOptionsSection elasticSearchOptions, ILogger logger) { var settings = new ConnectionSettings(new Uri(elasticSearchOptions.Uri!)) - .BasicAuthentication( - elasticSearchOptions.Username, - elasticSearchOptions.Password) - .MapVerenigingDocument(elasticSearchOptions.Indices!.Verenigingen!); + .BasicAuthentication( + elasticSearchOptions.Username, + elasticSearchOptions.Password) + .MapVerenigingDocument(elasticSearchOptions.Indices!.Verenigingen!) + .MapDuplicateDetectionDocument(elasticSearchOptions.Indices!.DuplicateDetection!); + + if (elasticSearchOptions.EnableDevelopmentLogs) + settings = settings.DisableDirectStreaming() + .PrettyJson() + .OnRequestCompleted(apiCallDetails => + { + if (apiCallDetails.RequestBodyInBytes != null) + logger.LogDebug( + message: "{HttpMethod} {Uri} \n {RequestBody}", + apiCallDetails.HttpMethod, + apiCallDetails.Uri, + Encoding.UTF8.GetString(apiCallDetails.RequestBodyInBytes)); + + if (apiCallDetails.ResponseBodyInBytes != null) + logger.LogDebug(message: "Response: {ResponseBody}", + Encoding.UTF8.GetString(apiCallDetails.ResponseBodyInBytes)); + }); - var elasticClient = new ElasticClient(settings); - return elasticClient; + return new ElasticClient(settings); } public static ConnectionSettings MapVerenigingDocument(this ConnectionSettings settings, string indexName) { return settings.DefaultMappingFor( typeof(VerenigingZoekDocument), - descriptor => descriptor.IndexName(indexName) - .IdProperty(nameof(VerenigingZoekDocument.VCode))); + selector: descriptor => descriptor.IndexName(indexName) + .IdProperty(nameof(VerenigingZoekDocument.VCode))); + } + + public static ConnectionSettings MapDuplicateDetectionDocument(this ConnectionSettings settings, string indexName) + { + return settings.DefaultMappingFor( + typeof(DuplicateDetectionDocument), + selector: descriptor => descriptor.IndexName(indexName) + .IdProperty(nameof(DuplicateDetectionDocument.VCode))); } } diff --git a/src/AssociationRegistry.Admin.Api/Program.cs b/src/AssociationRegistry.Admin.Api/Program.cs index f0c733578..612c91fc5 100755 --- a/src/AssociationRegistry.Admin.Api/Program.cs +++ b/src/AssociationRegistry.Admin.Api/Program.cs @@ -26,6 +26,7 @@ namespace AssociationRegistry.Admin.Api; using Infrastructure.Json; using Infrastructure.Middleware; using JasperFx.CodeGeneration; +using JasperFx.Core; using Kbo; using Lamar.Microsoft.DependencyInjection; using Magda; diff --git a/src/AssociationRegistry.Admin.Api/Verenigingen/Registreer/FeitelijkeVereniging/RequetsModels/RegistreerFeitelijkeVerenigingRequest.cs b/src/AssociationRegistry.Admin.Api/Verenigingen/Registreer/FeitelijkeVereniging/RequetsModels/RegistreerFeitelijkeVerenigingRequest.cs index 68360e8df..8d65272c4 100644 --- a/src/AssociationRegistry.Admin.Api/Verenigingen/Registreer/FeitelijkeVereniging/RequetsModels/RegistreerFeitelijkeVerenigingRequest.cs +++ b/src/AssociationRegistry.Admin.Api/Verenigingen/Registreer/FeitelijkeVereniging/RequetsModels/RegistreerFeitelijkeVerenigingRequest.cs @@ -14,19 +14,19 @@ public class RegistreerFeitelijkeVerenigingRequest /// Naam van de vereniging [DataMember] [Required] - public string Naam { get; init; } = null!; + public string Naam { get; set; } = null!; /// Korte naam van de vereniging [DataMember] - public string? KorteNaam { get; init; } + public string? KorteNaam { get; set; } /// Korte beschrijving van de vereniging [DataMember] - public string? KorteBeschrijving { get; init; } + public string? KorteBeschrijving { get; set; } /// Datum waarop de vereniging gestart is. Deze datum mag niet later zijn dan vandaag [DataMember] - public DateOnly? Startdatum { get; init; } + public DateOnly? Startdatum { get; set; } /// /// De doelgroep waar de activiteiten van deze vereniging zich op concentreert diff --git a/src/AssociationRegistry.Admin.Api/Verenigingen/Registreer/PotentialDuplicatesResponse.cs b/src/AssociationRegistry.Admin.Api/Verenigingen/Registreer/PotentialDuplicatesResponse.cs index 981fdc496..ec6c3adb4 100644 --- a/src/AssociationRegistry.Admin.Api/Verenigingen/Registreer/PotentialDuplicatesResponse.cs +++ b/src/AssociationRegistry.Admin.Api/Verenigingen/Registreer/PotentialDuplicatesResponse.cs @@ -1,11 +1,11 @@ namespace AssociationRegistry.Admin.Api.Verenigingen.Registreer; +using DuplicateVerenigingDetection; +using Infrastructure.ConfigurationBindings; using System; using System.Collections.Immutable; using System.Linq; using System.Runtime.Serialization; -using DuplicateVerenigingDetection; -using Infrastructure.ConfigurationBindings; [DataContract] public class PotentialDuplicatesResponse diff --git a/src/AssociationRegistry.Admin.Api/appsettings.development.json b/src/AssociationRegistry.Admin.Api/appsettings.development.json index 9aee94534..2d654b4df 100644 --- a/src/AssociationRegistry.Admin.Api/appsettings.development.json +++ b/src/AssociationRegistry.Admin.Api/appsettings.development.json @@ -16,7 +16,8 @@ "Username": "elastic", "Password": "local_development", "Indices": { - "Verenigingen": "verenigingsregister-verenigingen-admin" + "Verenigingen": "verenigingsregister-verenigingen-admin", + "DuplicateDetection": "verenigingsregister-duplicate-detection" } }, "PostgreSQLOptions": { diff --git a/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/ConfigurationBindings/ElasticSearchOptionsSection.cs b/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/ConfigurationBindings/ElasticSearchOptionsSection.cs index 2417d5361..115b72bb2 100644 --- a/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/ConfigurationBindings/ElasticSearchOptionsSection.cs +++ b/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/ConfigurationBindings/ElasticSearchOptionsSection.cs @@ -10,5 +10,6 @@ public class ElasticSearchOptionsSection public class IndicesOptionsSection { public string? Verenigingen { get; set; } + public string? DuplicateDetection { get; set; } } } diff --git a/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Extensions/ElasticClientExtensions.cs b/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Extensions/ElasticClientExtensions.cs index 697f9165a..055c6956c 100644 --- a/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Extensions/ElasticClientExtensions.cs +++ b/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Extensions/ElasticClientExtensions.cs @@ -1,14 +1,37 @@ namespace AssociationRegistry.Admin.ProjectionHost.Infrastructure.Extensions; -using Schema.Search; using Nest; using Nest.Specification.IndicesApi; +using Schema.Search; public static class ElasticClientExtensions { public static void CreateVerenigingIndex(this IndicesNamespace indicesNamespace, IndexName index) => indicesNamespace.Create( index, - descriptor => + selector: descriptor => descriptor.Map(VerenigingZoekDocumentMapping.Get)); + + public static void CreateDuplicateDetectionIndex(this IndicesNamespace indicesNamespace, IndexName index) + => indicesNamespace.Create( + index, + selector: c => c + .Settings(s => s + .Analysis(a => a + .Analyzers(AddDuplicateDetectionAnalyzer) + .TokenFilters(AddDutchStopWordsFilter))) + .Map(DuplicateDetectionDocumentMapping.Get)); + + private static TokenFiltersDescriptor AddDutchStopWordsFilter(TokenFiltersDescriptor tf) + => tf.Stop(name: "dutch_stop", selector: st => st + .StopWords("_dutch_") // Or provide your custom list + ); + + private static AnalyzersDescriptor AddDuplicateDetectionAnalyzer(AnalyzersDescriptor ad) + => ad.Custom(DuplicateDetectionDocumentMapping.DuplicateAnalyzer, + selector: ca + => ca + .Tokenizer("standard") + .Filters("lowercase", "asciifolding", "dutch_stop") + ); } diff --git a/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Extensions/ElasticSearchExtensions.cs b/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Extensions/ElasticSearchExtensions.cs index 471b2c114..22738161c 100644 --- a/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Extensions/ElasticSearchExtensions.cs +++ b/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Extensions/ElasticSearchExtensions.cs @@ -1,10 +1,8 @@ namespace AssociationRegistry.Admin.ProjectionHost.Infrastructure.Extensions; -using System; using ConfigurationBindings; -using Schema.Search; -using Microsoft.Extensions.DependencyInjection; using Nest; +using Schema.Search; public static class ElasticSearchExtensions { @@ -13,7 +11,10 @@ public static IServiceCollection AddElasticSearch( ElasticSearchOptionsSection elasticSearchOptions) { var elasticClient = CreateElasticClient(elasticSearchOptions); - EnsureIndexExists(elasticClient, elasticSearchOptions.Indices!.Verenigingen!); + + EnsureIndexExists(elasticClient, + elasticSearchOptions.Indices!.Verenigingen!, + elasticSearchOptions.Indices!.DuplicateDetection!); services.AddSingleton(_ => elasticClient); services.AddSingleton(_ => elasticClient); @@ -21,21 +22,29 @@ public static IServiceCollection AddElasticSearch( return services; } - private static void EnsureIndexExists(IElasticClient elasticClient, string verenigingenIndexName) + public static void EnsureIndexExists( + IElasticClient elasticClient, + string verenigingenIndexName, + string duplicateDetectionIndexName) { if (!elasticClient.Indices.Exists(verenigingenIndexName).Exists) elasticClient.Indices.CreateVerenigingIndex(verenigingenIndexName); + + if (!elasticClient.Indices.Exists(duplicateDetectionIndexName).Exists) + elasticClient.Indices.CreateDuplicateDetectionIndex(duplicateDetectionIndexName); } private static ElasticClient CreateElasticClient(ElasticSearchOptionsSection elasticSearchOptions) { var settings = new ConnectionSettings(new Uri(elasticSearchOptions.Uri!)) - .BasicAuthentication( - elasticSearchOptions.Username, - elasticSearchOptions.Password) - .MapVerenigingDocument(elasticSearchOptions.Indices!.Verenigingen!); + .BasicAuthentication( + elasticSearchOptions.Username, + elasticSearchOptions.Password) + .MapVerenigingDocument(elasticSearchOptions.Indices!.Verenigingen!) + .MapDuplicateDetectionDocument(elasticSearchOptions.Indices!.DuplicateDetection!); var elasticClient = new ElasticClient(settings); + return elasticClient; } @@ -43,7 +52,15 @@ public static ConnectionSettings MapVerenigingDocument(this ConnectionSettings s { return settings.DefaultMappingFor( typeof(VerenigingZoekDocument), - descriptor => descriptor.IndexName(indexName) - .IdProperty(nameof(VerenigingZoekDocument.VCode))); + selector: descriptor => descriptor.IndexName(indexName) + .IdProperty(nameof(VerenigingZoekDocument.VCode))); + } + + public static ConnectionSettings MapDuplicateDetectionDocument(this ConnectionSettings settings, string indexName) + { + return settings.DefaultMappingFor( + typeof(DuplicateDetectionDocument), + selector: descriptor => descriptor.IndexName(indexName) + .IdProperty(nameof(DuplicateDetectionDocument.VCode))); } } diff --git a/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureElasticSearchExtensions.cs b/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureElasticSearchExtensions.cs index f9fbd0657..ab31d4f1c 100644 --- a/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureElasticSearchExtensions.cs +++ b/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureElasticSearchExtensions.cs @@ -1,19 +1,20 @@ namespace AssociationRegistry.Admin.ProjectionHost.Infrastructure.Program.WebApplicationBuilder; -using System; using ConfigurationBindings; using Extensions; -using Microsoft.Extensions.DependencyInjection; using Nest; public static class ConfigureElasticSearchExtensions { - public static IServiceCollection ConfigureElasticSearch( + public static IServiceCollection ConfigureElasticSearch( this IServiceCollection services, ElasticSearchOptionsSection elasticSearchOptions) { var elasticClient = CreateElasticClient(elasticSearchOptions); - EnsureIndexExists(elasticClient, elasticSearchOptions.Indices!.Verenigingen!); + + ElasticSearchExtensions.EnsureIndexExists(elasticClient, + elasticSearchOptions.Indices!.Verenigingen!, + elasticSearchOptions.Indices!.DuplicateDetection!); services.AddSingleton(_ => elasticClient); services.AddSingleton(_ => elasticClient); @@ -21,21 +22,17 @@ public static IServiceCollection ConfigureElasticSearch( return services; } - private static void EnsureIndexExists(IElasticClient elasticClient, string verenigingenIndexName) - { - if (!elasticClient.Indices.Exists(verenigingenIndexName).Exists) - elasticClient.Indices.CreateVerenigingIndex(verenigingenIndexName); - } - private static ElasticClient CreateElasticClient(ElasticSearchOptionsSection elasticSearchOptions) { var settings = new ConnectionSettings(new Uri(elasticSearchOptions.Uri!)) - .BasicAuthentication( - elasticSearchOptions.Username, - elasticSearchOptions.Password) - .MapVerenigingDocument(elasticSearchOptions.Indices!.Verenigingen!); + .BasicAuthentication( + elasticSearchOptions.Username, + elasticSearchOptions.Password) + .MapVerenigingDocument(elasticSearchOptions.Indices!.Verenigingen!) + .MapDuplicateDetectionDocument(elasticSearchOptions.Indices.DuplicateDetection!); var elasticClient = new ElasticClient(settings); + return elasticClient; } } diff --git a/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureMartenExtensions.cs b/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureMartenExtensions.cs index 776b68122..581287fa5 100644 --- a/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureMartenExtensions.cs +++ b/src/AssociationRegistry.Admin.ProjectionHost/Infrastructure/Program/WebApplicationBuilder/ConfigureMartenExtensions.cs @@ -17,11 +17,13 @@ namespace AssociationRegistry.Admin.ProjectionHost.Infrastructure.Program.WebApp using Schema.Historiek; using System.Configuration; using Wolverine; -using ConfigurationManager = Microsoft.Extensions.Configuration.ConfigurationManager; +using ConfigurationManager = ConfigurationManager; public static class ConfigureMartenExtensions { - public static IServiceCollection ConfigureProjectionsWithMarten(this IServiceCollection source, ConfigurationManager configurationManager) + public static IServiceCollection ConfigureProjectionsWithMarten( + this IServiceCollection source, + ConfigurationManager configurationManager) { source .AddTransient(); diff --git a/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/DuplicateDetection/DuplicateDetectionProjectionHandler.cs b/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/DuplicateDetection/DuplicateDetectionProjectionHandler.cs new file mode 100644 index 000000000..be10b661e --- /dev/null +++ b/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/DuplicateDetection/DuplicateDetectionProjectionHandler.cs @@ -0,0 +1,109 @@ +namespace AssociationRegistry.Admin.ProjectionHost.Projections.Search.DuplicateDetection; + +using Events; +using Formatters; +using Schema.Search; +using Vereniging; + +public class DuplicateDetectionProjectionHandler +{ + private readonly IElasticRepository _elasticRepository; + + public DuplicateDetectionProjectionHandler(IElasticRepository elasticRepository) + { + _elasticRepository = elasticRepository; + } + + public async Task Handle(EventEnvelope message) + => await _elasticRepository.IndexAsync( + new DuplicateDetectionDocument + { + VCode = message.Data.VCode, + VerenigingsTypeCode = Verenigingstype.FeitelijkeVereniging.Code, + Naam = message.Data.Naam, + KorteNaam = message.Data.KorteNaam, + Locaties = message.Data.Locaties.Select(Map).ToArray(), + HoofdactiviteitVerenigingsloket = MapHoofdactiviteitVerenigingsloket(message.Data.HoofdactiviteitenVerenigingsloket), + } + ); + + public async Task Handle(EventEnvelope message) + => await _elasticRepository.IndexAsync( + new DuplicateDetectionDocument + { + VCode = message.Data.VCode, + VerenigingsTypeCode = Verenigingstype.Afdeling.Code, + Naam = message.Data.Naam, + KorteNaam = message.Data.KorteNaam, + Locaties = message.Data.Locaties.Select(Map).ToArray(), + HoofdactiviteitVerenigingsloket = MapHoofdactiviteitVerenigingsloket(message.Data.HoofdactiviteitenVerenigingsloket), + } + ); + + public async Task Handle(EventEnvelope message) + => await _elasticRepository.IndexAsync( + new DuplicateDetectionDocument + { + VCode = message.Data.VCode, + VerenigingsTypeCode = Verenigingstype.Parse(message.Data.Rechtsvorm).Code, + Naam = message.Data.Naam, + KorteNaam = message.Data.KorteNaam, + Locaties = Array.Empty(), + HoofdactiviteitVerenigingsloket = Array.Empty(), + } + ); + + public async Task Handle(EventEnvelope message) + => await _elasticRepository.UpdateAsync( + message.Data.VCode, + new DuplicateDetectionDocument + { + Naam = message.Data.Naam, + } + ); + + public async Task Handle(EventEnvelope message) + => await _elasticRepository.UpdateAsync( + message.Data.VCode, + new DuplicateDetectionDocument + { + KorteNaam = message.Data.KorteNaam, + } + ); + + public async Task Handle(EventEnvelope message) + => await _elasticRepository.UpdateAsync( + message.VCode, + new DuplicateDetectionDocument + { + HoofdactiviteitVerenigingsloket = MapHoofdactiviteitVerenigingsloket(message.Data.HoofdactiviteitenVerenigingsloket), + } + ); + + public async Task Handle(EventEnvelope message) + => await _elasticRepository.AppendLocatie(message.VCode, Map(message.Data.Locatie)); + + public async Task Handle(EventEnvelope message) + => await _elasticRepository.UpdateLocatie(message.VCode, Map(message.Data.Locatie)); + + public async Task Handle(EventEnvelope message) + => await _elasticRepository.RemoveLocatie(message.VCode, message.Data.Locatie.LocatieId); + + private static DuplicateDetectionDocument.Locatie Map(Registratiedata.Locatie locatie) + => new() + { + LocatieId = locatie.LocatieId, + Locatietype = locatie.Locatietype, + Naam = locatie.Naam, + Adresvoorstelling = locatie.Adres.ToAdresString(), + IsPrimair = locatie.IsPrimair, + Postcode = locatie.Adres?.Postcode ?? string.Empty, + Gemeente = locatie.Adres?.Gemeente ?? string.Empty, + }; + + private static string[] MapHoofdactiviteitVerenigingsloket( + IEnumerable hoofdactiviteitenVerenigingsloket) + { + return hoofdactiviteitenVerenigingsloket.Select(x => x.Code).ToArray(); + } +} diff --git a/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/ElasticRepository.cs b/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/ElasticRepository.cs index 17397a828..6cd08cfd9 100644 --- a/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/ElasticRepository.cs +++ b/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/ElasticRepository.cs @@ -50,9 +50,9 @@ public async Task UpdateAsync(string id, TDocument update) where TDoc throw new IndexDocumentFailed(response.DebugInformation); } - public async Task AppendLocatie(string id, VerenigingZoekDocument.Locatie locatie) + public async Task AppendLocatie(string id, ILocatie locatie) where T : class { - var response = await _elasticClient.UpdateAsync( + var response = await _elasticClient.UpdateAsync( id, selector: u => u.Script( s => s @@ -64,9 +64,9 @@ public async Task AppendLocatie(string id, VerenigingZoekDocument.Locatie locati throw new IndexDocumentFailed(response.DebugInformation); } - public async Task UpdateLocatie(string id, VerenigingZoekDocument.Locatie locatie) + public async Task UpdateLocatie(string id, ILocatie locatie) where T : class { - var response = await _elasticClient.UpdateAsync( + var response = await _elasticClient.UpdateAsync( id, selector: u => u.Script( s => s @@ -89,9 +89,9 @@ public async Task UpdateLocatie(string id, VerenigingZoekDocument.Locatie locati throw new IndexDocumentFailed(response.DebugInformation); } - public async Task RemoveLocatie(string id, int locatieId) + public async Task RemoveLocatie(string id, int locatieId) where T : class { - var response = await _elasticClient.UpdateAsync( + var response = await _elasticClient.UpdateAsync( id, selector: u => u.Script( s => s diff --git a/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/EventEnvelope.cs b/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/EventEnvelope.cs new file mode 100644 index 000000000..4254df3ae --- /dev/null +++ b/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/EventEnvelope.cs @@ -0,0 +1,22 @@ +namespace AssociationRegistry.Admin.ProjectionHost.Projections.Search; + +using Marten.Events; + +public class EventEnvelope : IEventEnvelope +{ + public string VCode + => Event.StreamKey!; + + public T Data + => (T)Event.Data; + + public Dictionary? Headers + => Event.Headers; + + public EventEnvelope(IEvent @event) + { + Event = @event; + } + + private IEvent Event { get; } +} diff --git a/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/IElasticRepository.cs b/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/IElasticRepository.cs index cc435d0d4..a5f61f9ae 100644 --- a/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/IElasticRepository.cs +++ b/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/IElasticRepository.cs @@ -13,7 +13,7 @@ Task IndexAsync(TDocument document) void Update(string id, TDocument update) where TDocument : class; Task UpdateAsync(string id, TDocument update) where TDocument : class; - Task AppendLocatie(string id, VerenigingZoekDocument.Locatie locatie); - Task RemoveLocatie(string id, int locatieId); - Task UpdateLocatie(string id, VerenigingZoekDocument.Locatie locatie); + Task AppendLocatie(string id, ILocatie locatie) where TDocument : class; + Task RemoveLocatie(string id, int locatieId) where TDocument : class; + Task UpdateLocatie(string id, ILocatie locatie) where TDocument : class; } diff --git a/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/IEventEnvelope.cs b/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/IEventEnvelope.cs new file mode 100644 index 000000000..282898cec --- /dev/null +++ b/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/IEventEnvelope.cs @@ -0,0 +1,5 @@ +namespace AssociationRegistry.Admin.ProjectionHost.Projections.Search; + +public interface IEventEnvelope +{ +} diff --git a/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/MartenEventsConsumer.cs b/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/MartenEventsConsumer.cs index bda55b041..f56e01c39 100644 --- a/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/MartenEventsConsumer.cs +++ b/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/MartenEventsConsumer.cs @@ -3,7 +3,6 @@ namespace AssociationRegistry.Admin.ProjectionHost.Projections.Search; using Marten.Events; using Wolverine; using Wolverine.Runtime.Routing; -using IEvent = Marten.Events.IEvent; public class MartenEventsConsumer : IMartenEventsConsumer { @@ -32,26 +31,3 @@ public async Task ConsumeAsync(IReadOnlyList streamActions) } } } - -public class EventEnvelope : IEventEnvelope -{ - public string VCode - => Event.StreamKey!; - - public T Data - => (T)Event.Data; - - public Dictionary? Headers - => Event.Headers; - - public EventEnvelope(IEvent @event) - { - Event = @event; - } - - private IEvent Event { get; } -} - -public interface IEventEnvelope -{ -} diff --git a/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/BeheerZoekProjection.cs b/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/Zoeken/BeheerZoekProjection.cs similarity index 96% rename from src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/BeheerZoekProjection.cs rename to src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/Zoeken/BeheerZoekProjection.cs index 0a54fb413..fd0140d06 100644 --- a/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/BeheerZoekProjection.cs +++ b/src/AssociationRegistry.Admin.ProjectionHost/Projections/Search/Zoeken/BeheerZoekProjection.cs @@ -1,4 +1,4 @@ -namespace AssociationRegistry.Admin.ProjectionHost.Projections.Search; +namespace AssociationRegistry.Admin.ProjectionHost.Projections.Search.Zoeken; using Events; using Formatters; @@ -187,21 +187,21 @@ await _elasticRepository.UpdateAsync( public async Task Handle(EventEnvelope message) { - await _elasticRepository.AppendLocatie( + await _elasticRepository.AppendLocatie( message.VCode, Map(message.Data.Locatie)); } public async Task Handle(EventEnvelope message) { - await _elasticRepository.UpdateLocatie( + await _elasticRepository.UpdateLocatie( message.VCode, Map(message.Data.Locatie)); } public async Task Handle(EventEnvelope message) { - await _elasticRepository.RemoveLocatie( + await _elasticRepository.RemoveLocatie( message.VCode, message.Data.Locatie.LocatieId); } @@ -227,14 +227,14 @@ private static Doelgroep Map(Registratiedata.Doelgroep doelgroep) public async Task Handle(EventEnvelope message) { - await _elasticRepository.AppendLocatie( + await _elasticRepository.AppendLocatie( message.VCode, Map(message.Data.Locatie)); } public async Task Handle(EventEnvelope message) { - await _elasticRepository.UpdateLocatie( + await _elasticRepository.UpdateLocatie( message.VCode, new VerenigingZoekDocument.Locatie { diff --git a/src/AssociationRegistry.Admin.ProjectionHost/appsettings.development.json b/src/AssociationRegistry.Admin.ProjectionHost/appsettings.development.json index 50e0baa7c..f534355ad 100644 --- a/src/AssociationRegistry.Admin.ProjectionHost/appsettings.development.json +++ b/src/AssociationRegistry.Admin.ProjectionHost/appsettings.development.json @@ -16,7 +16,8 @@ "Username": "elastic", "Password": "local_development", "Indices": { - "Verenigingen": "verenigingsregister-verenigingen-admin" + "Verenigingen": "verenigingsregister-verenigingen-admin", + "DuplicateDetection": "verenigingsregister-duplicate-detection" } }, "DistributedLock": { diff --git a/src/AssociationRegistry.Admin.Schema/Search/DuplicateDetectionDocument.cs b/src/AssociationRegistry.Admin.Schema/Search/DuplicateDetectionDocument.cs new file mode 100644 index 000000000..81ee48446 --- /dev/null +++ b/src/AssociationRegistry.Admin.Schema/Search/DuplicateDetectionDocument.cs @@ -0,0 +1,22 @@ +namespace AssociationRegistry.Admin.Schema.Search; + +public record DuplicateDetectionDocument +{ + public string VCode { get; set; } = null!; + public string Naam { get; set; } = null!; + public Locatie[] Locaties { get; set; } = null!; + public string VerenigingsTypeCode { get; set; } = null!; + public string KorteNaam { get; set; } = null!; + public string[] HoofdactiviteitVerenigingsloket { get; set; } = Array.Empty(); + + public record Locatie : ILocatie + { + public int LocatieId { get; init; } + public string Locatietype { get; init; } = null!; + public string? Naam { get; init; } + public string Adresvoorstelling { get; init; } = null!; + public bool IsPrimair { get; init; } + public string Postcode { get; init; } = null!; + public string Gemeente { get; init; } = null!; + } +} diff --git a/src/AssociationRegistry.Admin.Schema/Search/DuplicateDetectionDocumentMapping.cs b/src/AssociationRegistry.Admin.Schema/Search/DuplicateDetectionDocumentMapping.cs new file mode 100644 index 000000000..5183849f8 --- /dev/null +++ b/src/AssociationRegistry.Admin.Schema/Search/DuplicateDetectionDocumentMapping.cs @@ -0,0 +1,62 @@ +namespace AssociationRegistry.Admin.Schema.Search; + +using Nest; + +public static class DuplicateDetectionDocumentMapping +{ + public const string DuplicateAnalyzer = "duplicate_analyzer"; + + public static TypeMappingDescriptor Get(TypeMappingDescriptor map) + => map + .Properties( + descriptor => descriptor + .Keyword( + propertyDescriptor => propertyDescriptor + .Name(document => document.VCode)) + .Text( + propertyDescriptor => propertyDescriptor + .Name(document => document.Naam) + .Analyzer(DuplicateAnalyzer)) + .Text(propertyDescriptor => propertyDescriptor + .Name(document => document.KorteNaam) + ) + .Text(propertyDescriptor => propertyDescriptor + .Name(document => document.VerenigingsTypeCode) + ) + .Text(propertyDescriptor => propertyDescriptor + .Name(document => document.HoofdactiviteitVerenigingsloket)) + .Nested( + propertyDescriptor => propertyDescriptor + .Name(document => document.Locaties) + .IncludeInRoot() + .Properties(LocationMapping.Get)) + ); + + private static class LocationMapping + { + public static IPromise Get(PropertiesDescriptor map) + => map + .Text( + descriptor => descriptor + .Name(document => document.LocatieId)) + .Text( + propertyDescriptor => propertyDescriptor + .Name(document => document.Naam)) + .Text( + propertyDescriptor => propertyDescriptor + .Name(document => document.Adresvoorstelling)) + .Text( + propertyDescriptor => propertyDescriptor + .Name(document => document.IsPrimair)) + .Text( + propertyDescriptor => propertyDescriptor + .Name(document => document.Locatietype)) + .Text( + propertyDescriptor => propertyDescriptor + .Name(document => document.Postcode)) + .Text( + propertyDescriptor => propertyDescriptor + .Name(document => document.Gemeente) + .Analyzer(DuplicateAnalyzer)); + } +} diff --git a/src/AssociationRegistry.Admin.Schema/Search/ILocatie.cs b/src/AssociationRegistry.Admin.Schema/Search/ILocatie.cs new file mode 100644 index 000000000..56e842888 --- /dev/null +++ b/src/AssociationRegistry.Admin.Schema/Search/ILocatie.cs @@ -0,0 +1,6 @@ +namespace AssociationRegistry.Admin.Schema.Search; + +public interface ILocatie +{ + int LocatieId { get; } +} diff --git a/src/AssociationRegistry.Admin.Schema/Search/VerenigingZoekDocument.cs b/src/AssociationRegistry.Admin.Schema/Search/VerenigingZoekDocument.cs index d1a29d649..5dcac2131 100644 --- a/src/AssociationRegistry.Admin.Schema/Search/VerenigingZoekDocument.cs +++ b/src/AssociationRegistry.Admin.Schema/Search/VerenigingZoekDocument.cs @@ -14,7 +14,7 @@ public class VerenigingZoekDocument public Sleutel[] Sleutels { get; set; } = null!; public bool IsUitgeschrevenUitPubliekeDatastroom { get; set; } - public class Locatie + public class Locatie : ILocatie { public int LocatieId { get; init; } public string Locatietype { get; init; } = null!; diff --git a/src/AssociationRegistry/Events/HoofdactiviteitenVerenigingsloketWerdenGewijzigd.cs b/src/AssociationRegistry/Events/HoofdactiviteitenVerenigingsloketWerdenGewijzigd.cs index 0f31301c4..988402d96 100644 --- a/src/AssociationRegistry/Events/HoofdactiviteitenVerenigingsloketWerdenGewijzigd.cs +++ b/src/AssociationRegistry/Events/HoofdactiviteitenVerenigingsloketWerdenGewijzigd.cs @@ -1,17 +1,14 @@ namespace AssociationRegistry.Events; using Framework; +using Vereniging; -public record HoofdactiviteitenVerenigingsloketWerdenGewijzigd(HoofdactiviteitenVerenigingsloketWerdenGewijzigd.HoofdactiviteitVerenigingsloket[] HoofdactiviteitenVerenigingsloket) : IEvent +public record HoofdactiviteitenVerenigingsloketWerdenGewijzigd(Registratiedata.HoofdactiviteitVerenigingsloket[] HoofdactiviteitenVerenigingsloket) : IEvent { - public record HoofdactiviteitVerenigingsloket( - string Code, - string Beschrijving) - { - public static HoofdactiviteitVerenigingsloket With(Vereniging.HoofdactiviteitVerenigingsloket activiteit) - => new(activiteit.Code, activiteit.Beschrijving); - } - public static HoofdactiviteitenVerenigingsloketWerdenGewijzigd With(Vereniging.HoofdactiviteitVerenigingsloket[] hoofdactiviteitVerenigingslokets) - => new(hoofdactiviteitVerenigingslokets.Select(HoofdactiviteitVerenigingsloket.With).ToArray()); + public static HoofdactiviteitenVerenigingsloketWerdenGewijzigd With( + IEnumerable hoofdactiviteitenVerenigingsloket) + => new(hoofdactiviteitenVerenigingsloket + .Select(Registratiedata.HoofdactiviteitVerenigingsloket.With) + .ToArray()); } diff --git a/test/AssociationRegistry.Test.Acm.Api/appsettings.json b/test/AssociationRegistry.Test.Acm.Api/appsettings.json index 122a3a16f..d5a875236 100644 --- a/test/AssociationRegistry.Test.Acm.Api/appsettings.json +++ b/test/AssociationRegistry.Test.Acm.Api/appsettings.json @@ -34,7 +34,8 @@ "Username": "elastic", "Password": "local_development", "Indices": { - "Verenigingen": "test-verenigingsregister-verenigingen" + "Verenigingen": "test-verenigingsregister-verenigingen", + "DuplicateDetection": "test-verenigingsregister-duplicate-detection" } }, "PostgreSQLOptions": { diff --git a/test/AssociationRegistry.Test.Admin.Api/Afdeling/When_WijzigBasisGegevens/With_NaamGewijzigd.cs b/test/AssociationRegistry.Test.Admin.Api/Afdeling/When_WijzigBasisGegevens/With_NaamGewijzigd.cs index ba8051416..6dbcf4281 100644 --- a/test/AssociationRegistry.Test.Admin.Api/Afdeling/When_WijzigBasisGegevens/With_NaamGewijzigd.cs +++ b/test/AssociationRegistry.Test.Admin.Api/Afdeling/When_WijzigBasisGegevens/With_NaamGewijzigd.cs @@ -18,12 +18,12 @@ public sealed class When_NaamGewijzigd_Setup { public WijzigBasisgegevensRequest Request { get; } - public V047_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWijzigen Scenario { get; } + public V052_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWijzigen Scenario { get; } public HttpResponseMessage Response { get; } public When_NaamGewijzigd_Setup(EventsInDbScenariosFixture fixture) { - Scenario = fixture.V047AfdelingWerdGeregistreerdMetBestaandeMoederVoorNaamWijzigen; + Scenario = fixture.V052AfdelingWerdGeregistreerdMetBestaandeMoederVoorNaamWijzigen; Request = new Fixture().CustomizeAdminApi().Create(); diff --git a/test/AssociationRegistry.Test.Admin.Api/AssociationRegistry.Test.Admin.Api.csproj b/test/AssociationRegistry.Test.Admin.Api/AssociationRegistry.Test.Admin.Api.csproj index ce98ef277..c28690c60 100644 --- a/test/AssociationRegistry.Test.Admin.Api/AssociationRegistry.Test.Admin.Api.csproj +++ b/test/AssociationRegistry.Test.Admin.Api/AssociationRegistry.Test.Admin.Api.csproj @@ -36,19 +36,10 @@ - - Always - - - Always - - Always - - - + Always diff --git a/test/AssociationRegistry.Test.Admin.Api/Fixtures/AdminApiFixture.cs b/test/AssociationRegistry.Test.Admin.Api/Fixtures/AdminApiFixture.cs index 66699ed42..d610a5ef4 100644 --- a/test/AssociationRegistry.Test.Admin.Api/Fixtures/AdminApiFixture.cs +++ b/test/AssociationRegistry.Test.Admin.Api/Fixtures/AdminApiFixture.cs @@ -47,6 +47,9 @@ public AdminApiClient AdminApiClient private string VerenigingenIndexName => GetConfiguration()["ElasticClientOptions:Indices:Verenigingen"]; + private string DuplicateDetectionIndexName + => GetConfiguration()["ElasticClientOptions:Indices:DuplicateDetection"]; + protected AdminApiFixture() { WaitFor.PostGreSQLToBecomeAvailable( @@ -100,7 +103,7 @@ protected AdminApiFixture() builder.UseSetting(key: "ElasticClientOptions:Indices:Verenigingen", _identifier); }); - ConfigureElasticClient(ElasticClient, VerenigingenIndexName); + ConfigureElasticClient(ElasticClient, VerenigingenIndexName, DuplicateDetectionIndexName); } public IDocumentStore ApiDocumentStore @@ -160,13 +163,21 @@ private static void EnsureDbExists(IConfigurationRoot configuration) } } - private static void ConfigureElasticClient(IElasticClient client, string verenigingenIndexName) + private static void ConfigureElasticClient( + IElasticClient client, + string verenigingenIndexName, + string duplicateDetectionIndexName) { if (client.Indices.Exists(verenigingenIndexName).Exists) client.Indices.Delete(verenigingenIndexName); client.Indices.CreateVerenigingIndex(verenigingenIndexName); + if (client.Indices.Exists(duplicateDetectionIndexName).Exists) + client.Indices.Delete(duplicateDetectionIndexName); + + client.Indices.CreateDuplicateDetectionIndex(duplicateDetectionIndexName); + client.Indices.Refresh(Indices.All); } diff --git a/test/AssociationRegistry.Test.Admin.Api/Fixtures/EventsInDbScenariosFixture.cs b/test/AssociationRegistry.Test.Admin.Api/Fixtures/EventsInDbScenariosFixture.cs index d0811d7cb..498f36d4c 100644 --- a/test/AssociationRegistry.Test.Admin.Api/Fixtures/EventsInDbScenariosFixture.cs +++ b/test/AssociationRegistry.Test.Admin.Api/Fixtures/EventsInDbScenariosFixture.cs @@ -126,13 +126,16 @@ public readonly V045_VerenigingMetRechtspersoonlijkheidWerdGeregistreerd_With_Co public readonly V046_FeitelijkeVerenigingWerdGeregistreerd_ForWijzigStartdatum V046FeitelijkeVerenigingWerdGeregistreerdForWijzigStartdatum = new(); - public readonly V047_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWijzigen - V047AfdelingWerdGeregistreerdMetBestaandeMoederVoorNaamWijzigen = new(); + public readonly V052_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWijzigen + V052AfdelingWerdGeregistreerdMetBestaandeMoederVoorNaamWijzigen = new(); - public readonly V049_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWerdGewijzigd - V049AfdelingWerdGeregistreerdMetBestaandeMoederVoorNaamWerdGewijzigd = new(); + public readonly V054_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWerdGewijzigd + V054AfdelingWerdGeregistreerdMetBestaandeMoederVoorNaamWerdGewijzigd = new(); + public readonly V047_FeitelijkeVerenigingWerdGeregistreerd_WithMinimalFields_ForDuplicateDetection_WithAnalyzer + V047FeitelijkeVerenigingWerdGeregistreerdWithMinimalFieldsForDuplicateDetectionWithAnalyzer = new(); + protected override async Task Given() { var scenarios = new IEventsInDbScenario[] @@ -181,14 +184,19 @@ protected override async Task Given() V044VerenigingMetRechtspersoonlijkheidWerdGeregistreerdWithWijzigMaatschappelijkeZetelVolgensKbo, V045VerenigingMetRechtspersoonlijkheidWerdGeregistreerdWithContactgegevenFromKboForWijzigen, V046FeitelijkeVerenigingWerdGeregistreerdForWijzigStartdatum, - V047AfdelingWerdGeregistreerdMetBestaandeMoederVoorNaamWijzigen, - V049AfdelingWerdGeregistreerdMetBestaandeMoederVoorNaamWerdGewijzigd + V052AfdelingWerdGeregistreerdMetBestaandeMoederVoorNaamWijzigen, + V054AfdelingWerdGeregistreerdMetBestaandeMoederVoorNaamWerdGewijzigd, }; foreach (var scenario in scenarios) { scenario.Result = await AddEvents(scenario.VCode, scenario.GetEvents(), scenario.GetCommandMetadata()); } + + foreach (var (vCode, events) in V047FeitelijkeVerenigingWerdGeregistreerdWithMinimalFieldsForDuplicateDetectionWithAnalyzer.EventsPerVCode) + { + await AddEvents(vCode, events); + } } } diff --git a/test/AssociationRegistry.Test.Admin.Api/Fixtures/Scenarios/EventsInDb/V013_FeitelijkeVerenigingWerdGeregistreerd_WithAllFields_ForDuplicateCheck.cs b/test/AssociationRegistry.Test.Admin.Api/Fixtures/Scenarios/EventsInDb/V013_FeitelijkeVerenigingWerdGeregistreerd_WithAllFields_ForDuplicateCheck.cs index 798668fc1..9a218ecd7 100644 --- a/test/AssociationRegistry.Test.Admin.Api/Fixtures/Scenarios/EventsInDb/V013_FeitelijkeVerenigingWerdGeregistreerd_WithAllFields_ForDuplicateCheck.cs +++ b/test/AssociationRegistry.Test.Admin.Api/Fixtures/Scenarios/EventsInDb/V013_FeitelijkeVerenigingWerdGeregistreerd_WithAllFields_ForDuplicateCheck.cs @@ -1,10 +1,10 @@ namespace AssociationRegistry.Test.Admin.Api.Fixtures.Scenarios.EventsInDb; +using AssociationRegistry.Framework; +using AutoFixture; using Events; using EventStore; -using AssociationRegistry.Framework; using Framework; -using AutoFixture; public class V013_FeitelijkeVerenigingWerdGeregistreerd_WithAllFields_ForDuplicateCheck : IEventsInDbScenario { @@ -15,7 +15,8 @@ public V013_FeitelijkeVerenigingWerdGeregistreerd_WithAllFields_ForDuplicateChec { var fixture = new Fixture().CustomizeAdminApi(); VCode = "V9999013"; - Naam = "De absoluut coolste club"; + Naam = "De absoluute club die cool is"; + FeitelijkeVerenigingWerdGeregistreerd = fixture.Create() with { VCode = VCode, @@ -32,6 +33,7 @@ public V013_FeitelijkeVerenigingWerdGeregistreerd_WithAllFields_ForDuplicateChec IsPrimair = i == 0, }).ToArray(), }; + Metadata = fixture.Create() with { ExpectedVersion = null }; } diff --git a/test/AssociationRegistry.Test.Admin.Api/Fixtures/Scenarios/EventsInDb/V047_FeitelijkeVerenigingWerdGeregistreerd_WithMinimalFields_ForDuplicateDetection_WithAnalyzer.cs b/test/AssociationRegistry.Test.Admin.Api/Fixtures/Scenarios/EventsInDb/V047_FeitelijkeVerenigingWerdGeregistreerd_WithMinimalFields_ForDuplicateDetection_WithAnalyzer.cs new file mode 100644 index 000000000..31c44bd15 --- /dev/null +++ b/test/AssociationRegistry.Test.Admin.Api/Fixtures/Scenarios/EventsInDb/V047_FeitelijkeVerenigingWerdGeregistreerd_WithMinimalFields_ForDuplicateDetection_WithAnalyzer.cs @@ -0,0 +1,108 @@ +namespace AssociationRegistry.Test.Admin.Api.Fixtures.Scenarios.EventsInDb; + +using AssociationRegistry.Framework; +using AutoFixture; +using Events; +using EventStore; +using Framework; +using Vereniging; + +public class V047_FeitelijkeVerenigingWerdGeregistreerd_WithMinimalFields_ForDuplicateDetection_WithAnalyzer +{ + public readonly CommandMetadata Metadata; + public (VCode, IEvent[])[] EventsPerVCode { get; } + + public V047_FeitelijkeVerenigingWerdGeregistreerd_WithMinimalFields_ForDuplicateDetection_WithAnalyzer() + { + var fixture = new Fixture().CustomizeAdminApi(); + /** + * Hoofdletter ongevoelig → Vereniging = verEniging + * Spatie ongevoelig + * Leading spaces → Grote vereniging = Grote vereniging + * Trailing spaces → Grote vereniging = Grote vereniging + * Dubbele spaces → Grote vereniging = Grote vereniging + * Accent ongevoelig → Cafésport = Cafesport + * Leesteken ongevoelig → Sint-Servaas = Sint Servaas + * Functiewoorden weglaten → De pottestampers = Pottestampers - de, het, van, … idealiter is deze lijst configureerbaar + * Fuzzy search = kleine schrijfverschillen negeren. Deze zijn de elastic mogelijkheden: + * Ander karakter gebruiken → Uitnodiging = Uitnodiding + * 1 karakter minder → Vereniging = Verenging + * 1 karakter meer → Vereniging = Vereeniging + * 2 karakters van plaats wisselen → Pottestampers = Pottestapmers + **/ + + var verenigingWerdGeregistreerdOmTeWijzigen = VerenigingWerdGeregistreerd(fixture, naam: "XXX van Technologïeënthusiasten: Inováçie & Ëntwikkeling", vCode: "V9999047", + postcode: "9832", gemeente: "Neder-over-opper-onder-heembeek"); + + var locatie = fixture.Create(); + var locatieTeVerwijderen = fixture.Create(); + + verenigingWerdGeregistreerdOmTeWijzigen.Item2 = + verenigingWerdGeregistreerdOmTeWijzigen + .Item2 + .Concat(new IEvent[] + { + new NaamWerdGewijzigd("V9999047", "Vereniging van Technologïeënthusiasten: Inováçie & Ëntwikkeling"), + new KorteNaamWerdGewijzigd("V9999047", "Korte Naam Test"), + new HoofdactiviteitenVerenigingsloketWerdenGewijzigd( + HoofdactiviteitVerenigingsloket.All().Take(3) + .Select(Registratiedata.HoofdactiviteitVerenigingsloket.With) + .ToArray()), + new LocatieWerdToegevoegd(locatie), + new LocatieWerdGewijzigd(locatie with { Naam = "Erembodegem"}), + new LocatieWerdToegevoegd(locatieTeVerwijderen), + + }) + .ToArray(); + + EventsPerVCode = new[] + { + verenigingWerdGeregistreerdOmTeWijzigen, + VerenigingWerdGeregistreerd(fixture, naam: "Grote Vereniging", vCode: "V9999048", postcode: "9832", + gemeente: "Neder-over-opper-onder-heembeek"), + VerenigingWerdGeregistreerd(fixture, naam: "Cafésport", vCode: "V9999049", postcode: "8800", gemeente: "Rumbeke"), + VerenigingWerdGeregistreerd(fixture, naam: "Sint-Servaas", vCode: "V9999050", postcode: "8800", gemeente: "Roeselare"), + VerenigingWerdGeregistreerd(fixture, naam: "De pottestampers", vCode: "V9999051", postcode: "9830", + gemeente: "Heist-op-den-Berg"), + + }; + + Metadata = fixture.Create() with { ExpectedVersion = null }; + } + + private (VCode, IEvent[]) VerenigingWerdGeregistreerd(Fixture fixture, string naam, string vCode, string postcode, string gemeente) + => (VCode.Create(vCode), new IEvent[] + { + fixture.Create() with + { + VCode = vCode, + Naam = naam, + Locaties = new[] + { + fixture.Create() with + { + Adres = fixture.Create() + with + { + Straatnaam = fixture.Create(), + Huisnummer = fixture.Create(), + Postcode = postcode, + Gemeente = gemeente, + Land = fixture.Create(), + }, + }, + }, + KorteNaam = string.Empty, + Startdatum = null, + KorteBeschrijving = string.Empty, + Contactgegevens = Array.Empty(), + Vertegenwoordigers = Array.Empty(), + HoofdactiviteitenVerenigingsloket = Array.Empty(), + }, + }); + + public StreamActionResult Result { get; set; } = null!; + + public CommandMetadata GetCommandMetadata() + => Metadata; +} diff --git a/test/AssociationRegistry.Test.Admin.Api/Fixtures/Scenarios/EventsInDb/V047_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWijzigen.cs b/test/AssociationRegistry.Test.Admin.Api/Fixtures/Scenarios/EventsInDb/V052_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWijzigen.cs similarity index 91% rename from test/AssociationRegistry.Test.Admin.Api/Fixtures/Scenarios/EventsInDb/V047_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWijzigen.cs rename to test/AssociationRegistry.Test.Admin.Api/Fixtures/Scenarios/EventsInDb/V052_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWijzigen.cs index af32db858..39477f9f9 100644 --- a/test/AssociationRegistry.Test.Admin.Api/Fixtures/Scenarios/EventsInDb/V047_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWijzigen.cs +++ b/test/AssociationRegistry.Test.Admin.Api/Fixtures/Scenarios/EventsInDb/V052_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWijzigen.cs @@ -6,17 +6,17 @@ namespace AssociationRegistry.Test.Admin.Api.Fixtures.Scenarios.EventsInDb; using Framework; using AutoFixture; -public class V047_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWijzigen : IEventsInDbScenario +public class V052_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWijzigen : IEventsInDbScenario { public readonly VerenigingMetRechtspersoonlijkheidWerdGeregistreerd MoederWerdGeregistreerd; public readonly AfdelingWerdGeregistreerd AfdelingWerdGeregistreerd; public readonly CommandMetadata Metadata; - public V047_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWijzigen() + public V052_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWijzigen() { var fixture = new Fixture().CustomizeAdminApi(); - VCodeMoeder = "V9999047"; + VCodeMoeder = "V9999052"; NaamMoeder = "De coolste moeder"; MoederWerdGeregistreerd = fixture.Create() with { @@ -25,7 +25,7 @@ public V047_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWijzigen() }; KboNummerMoeder = MoederWerdGeregistreerd.KboNummer; - VCode = "V9999048"; + VCode = "V9999053"; AfdelingWerdGeregistreerd = fixture.Create() with { VCode = VCode, diff --git a/test/AssociationRegistry.Test.Admin.Api/Fixtures/Scenarios/EventsInDb/V049_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWerdGewijzigd.cs b/test/AssociationRegistry.Test.Admin.Api/Fixtures/Scenarios/EventsInDb/V054_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWerdGewijzigd.cs similarity index 93% rename from test/AssociationRegistry.Test.Admin.Api/Fixtures/Scenarios/EventsInDb/V049_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWerdGewijzigd.cs rename to test/AssociationRegistry.Test.Admin.Api/Fixtures/Scenarios/EventsInDb/V054_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWerdGewijzigd.cs index ace055e26..4ead2d798 100644 --- a/test/AssociationRegistry.Test.Admin.Api/Fixtures/Scenarios/EventsInDb/V049_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWerdGewijzigd.cs +++ b/test/AssociationRegistry.Test.Admin.Api/Fixtures/Scenarios/EventsInDb/V054_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWerdGewijzigd.cs @@ -7,18 +7,18 @@ namespace AssociationRegistry.Test.Admin.Api.Fixtures.Scenarios.EventsInDb; using AutoFixture; using Vereniging; -public class V049_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWerdGewijzigd : IEventsInDbScenario +public class V054_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWerdGewijzigd : IEventsInDbScenario { public readonly VerenigingMetRechtspersoonlijkheidWerdGeregistreerd MoederWerdGeregistreerd; public readonly AfdelingWerdGeregistreerd AfdelingWerdGeregistreerd; public readonly NaamWerdGewijzigd NaamWerdGewijzigd; public readonly CommandMetadata Metadata; - public V049_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWerdGewijzigd() + public V054_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWerdGewijzigd() { var fixture = new Fixture().CustomizeAdminApi(); - VCodeMoeder = "V9999049"; + VCodeMoeder = "V9999054"; NaamMoeder = "De coolste moeder"; VCode = "V9999050"; diff --git a/test/AssociationRegistry.Test.Admin.Api/WhenDetectingDuplicates/Given_Some_FeitelijkeVerenigingenGeregistreerd_Met_Gemeente.cs b/test/AssociationRegistry.Test.Admin.Api/WhenDetectingDuplicates/Given_Some_FeitelijkeVerenigingenGeregistreerd_Met_Gemeente.cs new file mode 100644 index 000000000..53867dbc8 --- /dev/null +++ b/test/AssociationRegistry.Test.Admin.Api/WhenDetectingDuplicates/Given_Some_FeitelijkeVerenigingenGeregistreerd_Met_Gemeente.cs @@ -0,0 +1,130 @@ +namespace AssociationRegistry.Test.Admin.Api.WhenDetectingDuplicates; + +using AssociationRegistry.Admin.Api.Verenigingen.Common; +using AssociationRegistry.Admin.Api.Verenigingen.Registreer.FeitelijkeVereniging.RequetsModels; +using AutoFixture; +using Events; +using Fixtures; +using Fixtures.Scenarios.EventsInDb; +using FluentAssertions; +using Framework; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System.Net; +using Vereniging; +using Xunit; +using Xunit.Categories; +using Adres = AssociationRegistry.Admin.Api.Verenigingen.Common.Adres; + +[Collection(nameof(AdminApiCollection))] +[Category("AdminApi")] +[IntegrationTest] +public class Given_Some_FeitelijkeVerenigingenGeregistreerd_Met_Gemeente +{ + private readonly AdminApiClient _adminApiClient; + private readonly Fixture _fixture; + private readonly V047_FeitelijkeVerenigingWerdGeregistreerd_WithMinimalFields_ForDuplicateDetection_WithAnalyzer _scenario; + + public Given_Some_FeitelijkeVerenigingenGeregistreerd_Met_Gemeente(EventsInDbScenariosFixture fixture) + { + _fixture = new Fixture().CustomizeAdminApi(); + _adminApiClient = fixture.AdminApiClient; + _scenario = fixture.V047FeitelijkeVerenigingWerdGeregistreerdWithMinimalFieldsForDuplicateDetectionWithAnalyzer; + } + + [Theory] + [InlineData("V9999047", "NEDER-OVER-OPPER-ONDER-HEEMBEEK")] + [InlineData("V9999047", "NeDeR-OvEr-oPpEr-OnDeR-HeEmBeEk")] + public async Task? Then_A_DuplicateIsDetected_WithDifferentCapitalization( + string duplicatesShouldContainThisVCode, + string verbasterdeGemeente) + { + await VerifyThatDuplicateIsFound(duplicatesShouldContainThisVCode, verbasterdeGemeente); + } + + [Theory] + [InlineData("V9999047", "Neder over opper onder heembeek")] + [InlineData("V9999047", "Neder over opper-onder-heembeek")] + [InlineData("V9999047", "Neder over opper. onder heembeek")] + [InlineData("V9999047", "Neder over opper-onder. heembeek")] + public async Task Then_A_DuplicateIsDetected_WithDifferentPunctuation( + string duplicatesShouldContainThisVCode, + string verbasterdeGemeente) + { + await VerifyThatDuplicateIsFound(duplicatesShouldContainThisVCode, verbasterdeGemeente); + } + + [Theory] + [InlineData("V9999047", "Neders-met-opper-onder-heembeek")] + [InlineData("V9999047", "Neder-over-van-onder-heembeek")] + [InlineData("V9999048", "Neder-met-opper-en-het-onder-heembeek")] + public async Task? Then_A_DuplicateIsDetected_WithStopwoorden(string duplicatesShouldContainThisVCode, string verbasterdeGemeente) + { + await VerifyThatDuplicateIsFound(duplicatesShouldContainThisVCode, verbasterdeGemeente); + } + + [Theory] + [InlineData("V9999047", "Neders-over-opper-onder-heembeek")] + [InlineData("V9999047", "Neder-over-oper-onder-heembeek")] + [InlineData("V9999048", "Neder-over-opper-onder-humbeek")] + public async Task? Then_A_DuplicateIsDetected_WithFoezieSearch( + string duplicatesShouldContainThisVCode, + string verbasterdeGemeente) + { + await VerifyThatDuplicateIsFound(duplicatesShouldContainThisVCode, verbasterdeGemeente); + } + + private async Task VerifyThatDuplicateIsFound(string duplicatesShouldContainThisVCode, string verbasterdeGemeente) + { + var request = CreateRegistreerFeitelijkeVerenigingRequest(duplicatesShouldContainThisVCode, verbasterdeGemeente); + + var response = await _adminApiClient.RegistreerFeitelijkeVereniging(JsonConvert.SerializeObject(request)); + response.StatusCode.Should().Be(HttpStatusCode.Conflict); + + var responseContent = await response.Content.ReadAsStringAsync(); + var duplicates = ExtractDuplicateVCode(responseContent); + duplicates.Should().Contain(duplicatesShouldContainThisVCode); + } + + private RegistreerFeitelijkeVerenigingRequest CreateRegistreerFeitelijkeVerenigingRequest( + string vCode, + string gemeeente) + { + var @event = _scenario.EventsPerVCode + .Single(t => t.Item1.Value == vCode) + .Item2 + .First() as FeitelijkeVerenigingWerdGeregistreerd; + + return new RegistreerFeitelijkeVerenigingRequest + { + Naam = @event.Naam, + Startdatum = DateOnly.FromDateTime(DateTime.Now), + KorteNaam = "", + KorteBeschrijving = "", + Locaties = new[] + { + new ToeTeVoegenLocatie + { + Locatietype = Locatietype.Correspondentie, + Adres = new Adres + { + Straatnaam = _fixture.Create(), + Huisnummer = _fixture.Create(), + Postcode = @event.Locaties.First().Adres.Postcode, + Gemeente = gemeeente, + Land = _fixture.Create(), + }, + }, + }, + }; + } + + private static IEnumerable ExtractDuplicateVCode(string responseContent) + { + var duplicates = JObject.Parse(responseContent) + .SelectTokens("$.mogelijkeDuplicateVerenigingen[*].vCode") + .Select(x => x.ToString()); + + return duplicates; + } +} diff --git a/test/AssociationRegistry.Test.Admin.Api/WhenDetectingDuplicates/Given_Some_FeitelijkeVerenigingenGeregistreerd_Met_Naam.cs b/test/AssociationRegistry.Test.Admin.Api/WhenDetectingDuplicates/Given_Some_FeitelijkeVerenigingenGeregistreerd_Met_Naam.cs new file mode 100644 index 000000000..666be213a --- /dev/null +++ b/test/AssociationRegistry.Test.Admin.Api/WhenDetectingDuplicates/Given_Some_FeitelijkeVerenigingenGeregistreerd_Met_Naam.cs @@ -0,0 +1,161 @@ +namespace AssociationRegistry.Test.Admin.Api.WhenDetectingDuplicates; + +using AssociationRegistry.Admin.Api.Verenigingen.Common; +using AssociationRegistry.Admin.Api.Verenigingen.Registreer.FeitelijkeVereniging.RequetsModels; +using AutoFixture; +using Events; +using Fixtures; +using Fixtures.Scenarios.EventsInDb; +using FluentAssertions; +using Framework; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System.Net; +using Vereniging; +using Xunit; +using Xunit.Categories; +using Adres = AssociationRegistry.Admin.Api.Verenigingen.Common.Adres; + +[Collection(nameof(AdminApiCollection))] +[Category("AdminApi")] +[IntegrationTest] +public class Given_Some_FeitelijkeVerenigingenGeregistreerd_Met_Naam +{ + private readonly AdminApiClient _adminApiClient; + private readonly Fixture _fixture; + private readonly V047_FeitelijkeVerenigingWerdGeregistreerd_WithMinimalFields_ForDuplicateDetection_WithAnalyzer _scenario; + + public Given_Some_FeitelijkeVerenigingenGeregistreerd_Met_Naam(EventsInDbScenariosFixture fixture) + { + _fixture = new Fixture().CustomizeAdminApi(); + _adminApiClient = fixture.AdminApiClient; + _scenario = fixture.V047FeitelijkeVerenigingWerdGeregistreerdWithMinimalFieldsForDuplicateDetectionWithAnalyzer; + } + + [Theory] + [InlineData("V9999048", "Grote vereniging")] + [InlineData("V9999048", "GROTE VERENIGING")] + [InlineData("V9999048", "grote vereniging")] + [InlineData("V9999051", "De Pottestampers")] + [InlineData("V9999051", "DE POTTESTAMPERS")] + [InlineData("V9999051", "de pottestampers")] + public async Task? Then_A_DuplicateIsDetected_WithDifferentCapitalization( + string duplicatesShouldContainThisVCode, + string verbasterdeNaam) + { + await VerifyThatDuplicateIsFound(duplicatesShouldContainThisVCode, verbasterdeNaam); + } + + [Theory] + [InlineData("V9999048", "Grote-Vereniging")] + [InlineData("V9999050", "Sint Servaas")] + [InlineData("V9999048", "Grote Vereniging!")] + [InlineData("V9999050", "Sint-Servaas!")] + [InlineData("V9999048", "Grote-Vereniging!")] + [InlineData("V9999050", "Sint Servaas!")] + public async Task Then_A_DuplicateIsDetected_WithDifferentPunctuation(string duplicatesShouldContainThisVCode, string verbasterdeNaam) + { + await VerifyThatDuplicateIsFound(duplicatesShouldContainThisVCode, verbasterdeNaam); + } + + [Theory] + [InlineData("V9999048", " Grote Vereniging")] + [InlineData("V9999048", "Grote Vereniging ")] + [InlineData("V9999048", "Grote Vereniging")] + [InlineData("V9999048", " Grote Vereniging ")] + public async Task? Then_A_DuplicateIsDetected_WithAdditionalSpaces(string duplicatesShouldContainThisVCode, string verbasterdeNaam) + { + await VerifyThatDuplicateIsFound(duplicatesShouldContainThisVCode, verbasterdeNaam); + } + + [Theory] + [InlineData("V9999047", "Vereniging van Technologieenthousiasten: Innovacie & Ontwikkeling")] + [InlineData("V9999049", "Cafesport")] + public async Task? Then_A_DuplicateIsDetected_WithNoAccents(string duplicatesShouldContainThisVCode, string verbasterdeNaam) + { + await VerifyThatDuplicateIsFound(duplicatesShouldContainThisVCode, verbasterdeNaam); + } + + [Theory] + [InlineData("V9999048", "Grote Veréniging")] + [InlineData("V9999051", "Dé pottestampers")] + public async Task? Then_A_DuplicateIsDetected_WithMoreAccents(string duplicatesShouldContainThisVCode, string verbasterdeNaam) + { + await VerifyThatDuplicateIsFound(duplicatesShouldContainThisVCode, verbasterdeNaam); + } + + [Theory] + [InlineData("V9999048", "De Grote van de Vereniging")] + [InlineData("V9999051", "pottestampers met het")] + public async Task? Then_A_DuplicateIsDetected_WithStopwoorden(string duplicatesShouldContainThisVCode, string verbasterdeNaam) + { + await VerifyThatDuplicateIsFound(duplicatesShouldContainThisVCode, verbasterdeNaam); + } + + [Theory] + [InlineData("V9999048", "Gorte Veregigning")] + [InlineData("V9999048", "Gorte Vereeegigning")] + [InlineData("V9999051", "De potenstampers")] + public async Task? Then_A_DuplicateIsDetected_WithFoezieSearch(string duplicatesShouldContainThisVCode, string verbasterdeNaam) + { + await VerifyThatDuplicateIsFound(duplicatesShouldContainThisVCode, verbasterdeNaam); + } + + private async Task VerifyThatDuplicateIsFound(string duplicatesShouldContainThisVCode, string verbasterdeNaam) + { + var request = FeitelijkeVerenigingWerdGeregistreerd(duplicatesShouldContainThisVCode, verbasterdeNaam); + + var response = await _adminApiClient.RegistreerFeitelijkeVereniging(JsonConvert.SerializeObject(request)); + response.StatusCode.Should().Be(HttpStatusCode.Conflict); + + var responseContent = await response.Content.ReadAsStringAsync(); + var duplicates = ExtractDuplicateVCode(responseContent); + duplicates.Should().Contain(duplicatesShouldContainThisVCode); + } + + private RegistreerFeitelijkeVerenigingRequest FeitelijkeVerenigingWerdGeregistreerd( + string vCode, + string naam) + { + var @event = _scenario.EventsPerVCode + .Single(t => t.Item1.Value == vCode) + .Item2 + .First() as FeitelijkeVerenigingWerdGeregistreerd; + + var request1 = new RegistreerFeitelijkeVerenigingRequest + { + Naam = naam, + Startdatum = DateOnly.FromDateTime(DateTime.Now), + KorteNaam = "", + KorteBeschrijving = "", + Locaties = new[] + { + new ToeTeVoegenLocatie + { + Locatietype = Locatietype.Correspondentie, + Adres = new Adres + { + Straatnaam = _fixture.Create(), + Huisnummer = _fixture.Create(), + Postcode = @event.Locaties.First().Adres.Postcode, + Gemeente = @event.Locaties.First().Adres.Gemeente, + Land = _fixture.Create(), + }, + }, + }, + }; + + var request = request1; + + return request; + } + + private static IEnumerable ExtractDuplicateVCode(string responseContent) + { + var duplicates = JObject.Parse(responseContent) + .SelectTokens("$.mogelijkeDuplicateVerenigingen[*].vCode") + .Select(x => x.ToString()); + + return duplicates; + } +} diff --git a/test/AssociationRegistry.Test.Admin.Api/WhenDetectingDuplicates/Given_Some_Updates.cs b/test/AssociationRegistry.Test.Admin.Api/WhenDetectingDuplicates/Given_Some_Updates.cs new file mode 100644 index 000000000..cdf81f8e9 --- /dev/null +++ b/test/AssociationRegistry.Test.Admin.Api/WhenDetectingDuplicates/Given_Some_Updates.cs @@ -0,0 +1,104 @@ +namespace AssociationRegistry.Test.Admin.Api.WhenDetectingDuplicates; + +using AssociationRegistry.Admin.Api.Verenigingen.Common; +using AssociationRegistry.Admin.Api.Verenigingen.Registreer.FeitelijkeVereniging.RequetsModels; +using AutoFixture; +using Events; +using Fixtures; +using Fixtures.Scenarios.EventsInDb; +using FluentAssertions; +using Framework; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System.Net; +using Vereniging; +using Xunit; +using Xunit.Categories; +using Adres = AssociationRegistry.Admin.Api.Verenigingen.Common.Adres; + +[Collection(nameof(AdminApiCollection))] +[Category("AdminApi")] +[IntegrationTest] +public class Given_Some_Updates +{ + private readonly AdminApiClient _adminApiClient; + private readonly Fixture _fixture; + private readonly V047_FeitelijkeVerenigingWerdGeregistreerd_WithMinimalFields_ForDuplicateDetection_WithAnalyzer _scenario; + + public Given_Some_Updates(EventsInDbScenariosFixture fixture) + { + _fixture = new Fixture().CustomizeAdminApi(); + _adminApiClient = fixture.AdminApiClient; + _scenario = fixture.V047FeitelijkeVerenigingWerdGeregistreerdWithMinimalFieldsForDuplicateDetectionWithAnalyzer; + } + + [Theory] + [InlineData("V9999047", "Vereniging van Technologïeënthusiasten: Inováçie & Ëntwikkeling")] + public async Task? Then_A_DuplicateIsDetected_WithDifferentCapitalization( + string duplicatesShouldContainThisVCode, + string naam) + { + var request = CreateRegistreerFeitelijkeVerenigingRequest(duplicatesShouldContainThisVCode, naam); + + var response = await _adminApiClient.RegistreerFeitelijkeVereniging(JsonConvert.SerializeObject(request)); + response.StatusCode.Should().Be(HttpStatusCode.Conflict); + + var responseContent = await response.Content.ReadAsStringAsync(); + + var duplicate = JObject.Parse(responseContent)["mogelijkeDuplicateVerenigingen"] + .Single(x => x["vCode"].Value() == duplicatesShouldContainThisVCode); + + duplicate["korteNaam"].Value().Should().Be("Korte Naam Test"); + } + + private RegistreerFeitelijkeVerenigingRequest CreateRegistreerFeitelijkeVerenigingRequest(string vCode, string naam) + { + var @event = _scenario.EventsPerVCode + .Single(t => t.Item1.Value == vCode) + .Item2 + .First() as FeitelijkeVerenigingWerdGeregistreerd; + + return new RegistreerFeitelijkeVerenigingRequest + { + Naam = naam, + Startdatum = DateOnly.FromDateTime(DateTime.Now), + KorteNaam = "", + KorteBeschrijving = "", + Locaties = new[] + { + new ToeTeVoegenLocatie + { + Locatietype = Locatietype.Correspondentie, + Adres = new Adres + { + Straatnaam = _fixture.Create(), + Huisnummer = _fixture.Create(), + Postcode = @event.Locaties.First().Adres.Postcode, + Gemeente = @event.Locaties.First().Adres.Gemeente, + Land = _fixture.Create(), + }, + }, + }, + }; + } + + private static IEnumerable ExtractDuplicateVCode(string responseContent) + => JObject.Parse(responseContent) + .SelectTokens("$.mogelijkeDuplicateVerenigingen[*].vCode") + .Select(x => x.ToString()); + + private static IEnumerable ExtractDuplicateKorteNaam(string responseContent) + => JObject.Parse(responseContent) + .SelectTokens("$.mogelijkeDuplicateVerenigingen[*].vCode") + .Select(x => x.ToString()); + + private static IEnumerable ExtractDuplicateNaam(string responseContent) + => JObject.Parse(responseContent) + .SelectTokens("$.mogelijkeDuplicateVerenigingen[*].vCode") + .Select(x => x.ToString()); + + private static IEnumerable ExtractDuplicateGemeentes(string responseContent) + => JObject.Parse(responseContent) + .SelectTokens("$.mogelijkeDuplicateVerenigingen[*].vCode") + .Select(x => x.ToString()); +} diff --git a/test/AssociationRegistry.Test.Admin.Api/WhenDetectingDuplicates/Playground/ForDuplicateDetection.cs b/test/AssociationRegistry.Test.Admin.Api/WhenDetectingDuplicates/Playground/ForDuplicateDetection.cs new file mode 100644 index 000000000..c6a2ef29c --- /dev/null +++ b/test/AssociationRegistry.Test.Admin.Api/WhenDetectingDuplicates/Playground/ForDuplicateDetection.cs @@ -0,0 +1,147 @@ +namespace AssociationRegistry.Test.Admin.Api.WhenDetectingDuplicates.Playground; + +using AssociationRegistry.Admin.Api.DuplicateDetection; +using AssociationRegistry.Admin.ProjectionHost.Infrastructure.ConfigurationBindings; +using AssociationRegistry.Admin.ProjectionHost.Infrastructure.Extensions; +using AssociationRegistry.Admin.ProjectionHost.Projections.Search; +using AssociationRegistry.Admin.Schema.Search; +using AssociationRegistry.Test.Admin.Api.Fixtures; +using AssociationRegistry.Vereniging; +using FluentAssertions; +using Microsoft.Extensions.Configuration; +using Nest; +using System.Reflection; +using Xunit; +using ElasticSearchExtensions = AssociationRegistry.Admin.ProjectionHost.Infrastructure.Extensions.ElasticSearchExtensions; + +public class ForDuplicateDetection : IClassFixture +{ + private readonly ElasticClient _elasticClient; + private readonly SearchDuplicateVerenigingDetectionService _duplicateDetectionService; + + public ForDuplicateDetection(DuplicateDetectionSetup setup) + { + _elasticClient = setup.Client; + _duplicateDetectionService = setup.DuplicateDetectionService; + } + + [Fact] + public async Task It_asciis_the_tokens() + { + var analyzeResponse = + await _elasticClient.Indices + .AnalyzeAsync(a => a + .Index() + .Analyzer(DuplicateDetectionDocumentMapping.DuplicateAnalyzer) + .Text( + "Vereniging van Technologïeënthusiasten: Inováçie & Ëntwikkeling") + ); + + analyzeResponse + .Tokens + .Select(x => x.Token) + .Should() + .BeEquivalentTo( + "vereniging", + "technologieenthusiasten", + "inovacie", + "entwikkeling"); + } + + [Fact] + public async Task Stopwords_Everywhere() + { + await _elasticClient.Indices.RefreshAsync(Indices.AllIndices); + + var duplicates = await _duplicateDetectionService.GetDuplicates( + VerenigingsNaam.Create("De Vereniging van de Technologïeënthusiasten en ook de Inováçie en van de Ëntwikkeling"), + Met1MatchendeGemeente()); + + duplicates.Should().HaveCount(1); + duplicates.Single().Naam.Should().Be("Vereniging van Technologïeënthusiasten: Inováçie & Ëntwikkeling"); + } + + [Fact] + public async Task Fuzzy() + { + await _elasticClient.Indices.RefreshAsync(Indices.AllIndices); + + var duplicates = await _duplicateDetectionService.GetDuplicates( + VerenigingsNaam.Create("De Verengiging vn Technologïeënthusiasten: Inováçie & Ëntwikkeling"), + Met1MatchendeGemeente()); + + duplicates.Should().HaveCount(1); + duplicates.Single().Naam.Should().Be("Vereniging van Technologïeënthusiasten: Inováçie & Ëntwikkeling"); + } + + + private static Locatie[] Met1MatchendeGemeente() + { + return new[] + { + Locatie.Create(naam: "xx", isPrimair: false, Locatietype.Correspondentie, adresId: null, + Adres.Create(straatnaam: "xx", huisnummer: "xx", busnummer: "xx", postcode: "xx", gemeente: "Hulste", + land: "xx")), + Locatie.Create(naam: "xx", isPrimair: false, Locatietype.Correspondentie, adresId: null, + Adres.Create(straatnaam: "xx", huisnummer: "xx", busnummer: "xx", postcode: "xx", gemeente: "Kortrijk", + land: "xx")), + }; + } +} + +public class DuplicateDetectionSetup +{ + public ElasticClient Client { get; } + public ElasticRepository Repository { get; } + public SearchDuplicateVerenigingDetectionService DuplicateDetectionService { get; } + + public DuplicateDetectionSetup() + { + var maybeRootDirectory = Directory + .GetParent(typeof(AdminApiFixture).GetTypeInfo().Assembly.Location)?.Parent?.Parent?.Parent?.FullName; + + if (maybeRootDirectory is not { } rootDirectory) + throw new NullReferenceException("Root directory cannot be null"); + + var configuration = new ConfigurationBuilder() + .SetBasePath(rootDirectory) + .AddJsonFile("appsettings.json") + .AddJsonFile($"appsettings.{Environment.MachineName.ToLower()}.json", optional: true) + .Build(); + + var elasticSearchOptions = configuration.GetSection("ElasticClientOptions") + .Get(); + + var duplicateDetection = "analyzethis"; + + var settings = new ConnectionSettings(new Uri(elasticSearchOptions.Uri!)) + .BasicAuthentication( + elasticSearchOptions.Username, + elasticSearchOptions.Password) + .MapVerenigingDocument(elasticSearchOptions.Indices!.Verenigingen!) + .MapDuplicateDetectionDocument(duplicateDetection); + + Client = new ElasticClient(settings); + Client.Indices.Delete(duplicateDetection); + ElasticSearchExtensions.EnsureIndexExists(Client, elasticSearchOptions.Indices.Verenigingen!, duplicateDetection); + + Repository = new ElasticRepository(Client); + + DuplicateDetectionService = new SearchDuplicateVerenigingDetectionService(Client); + + Repository.Index(new DuplicateDetectionDocument + { + VCode = "V0001001", + Naam = "Vereniging van Technologïeënthusiasten: Inováçie & Ëntwikkeling", + VerenigingsTypeCode = Verenigingstype.FeitelijkeVereniging.Code, + Locaties = new[] + { + new DuplicateDetectionDocument.Locatie + { + Gemeente = "Hulste", + Postcode = "8531", + }, + }, + }); + } +} diff --git a/test/AssociationRegistry.Test.Admin.Api/When_Retrieving_Detail/Given_MoederWerdGeregistreerd_And_Then_AfdelingWerdGeregistreerd_AndThen_NaamWerdGewijzigd.cs b/test/AssociationRegistry.Test.Admin.Api/When_Retrieving_Detail/Given_MoederWerdGeregistreerd_And_Then_AfdelingWerdGeregistreerd_AndThen_NaamWerdGewijzigd.cs index 7599fb0e0..806324c96 100644 --- a/test/AssociationRegistry.Test.Admin.Api/When_Retrieving_Detail/Given_MoederWerdGeregistreerd_And_Then_AfdelingWerdGeregistreerd_AndThen_NaamWerdGewijzigd.cs +++ b/test/AssociationRegistry.Test.Admin.Api/When_Retrieving_Detail/Given_MoederWerdGeregistreerd_And_Then_AfdelingWerdGeregistreerd_AndThen_NaamWerdGewijzigd.cs @@ -16,11 +16,11 @@ namespace AssociationRegistry.Test.Admin.Api.When_Retrieving_Detail; public class Given_MoederWerdGeregistreerd_And_Then_AfdelingWerdGeregistreerd_AndThen_NaamWerdGewijzigd { private readonly AdminApiClient _adminApiClient; - private readonly V049_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWerdGewijzigd _scenario; + private readonly V054_AfdelingWerdGeregistreerd_MetBestaandeMoeder_VoorNaamWerdGewijzigd _scenario; public Given_MoederWerdGeregistreerd_And_Then_AfdelingWerdGeregistreerd_AndThen_NaamWerdGewijzigd(EventsInDbScenariosFixture fixture) { - _scenario = fixture.V049AfdelingWerdGeregistreerdMetBestaandeMoederVoorNaamWerdGewijzigd; + _scenario = fixture.V054AfdelingWerdGeregistreerdMetBestaandeMoederVoorNaamWerdGewijzigd; _adminApiClient = fixture.DefaultClient; } diff --git a/test/AssociationRegistry.Test.Admin.Api/appsettings.json b/test/AssociationRegistry.Test.Admin.Api/appsettings.json index a778bd4f6..1d4ca2eca 100644 --- a/test/AssociationRegistry.Test.Admin.Api/appsettings.json +++ b/test/AssociationRegistry.Test.Admin.Api/appsettings.json @@ -15,7 +15,8 @@ "Username": "elastic", "Password": "local_development", "Indices": { - "Verenigingen": "test-verenigingsregister-admin-verenigingen" + "Verenigingen": "test-verenigingsregister-admin-verenigingen", + "DuplicateDetection": "test-verenigingsregister-duplicate-detection" } }, "PostgreSQLOptions": { diff --git a/test/AssociationRegistry.Test.Public.Api/appsettings.json b/test/AssociationRegistry.Test.Public.Api/appsettings.json index 943c86971..d9efa8737 100644 --- a/test/AssociationRegistry.Test.Public.Api/appsettings.json +++ b/test/AssociationRegistry.Test.Public.Api/appsettings.json @@ -34,7 +34,8 @@ "Username": "elastic", "Password": "local_development", "Indices": { - "Verenigingen": "test-verenigingsregister-verenigingen" + "Verenigingen": "test-verenigingsregister-verenigingen", + "DuplicateDetection": "test-verenigingsregister-duplicate-detection" } }, "PostgreSQLOptions": {