Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test: or-2582 projections tests without starting program #1021

Merged
merged 19 commits into from
Dec 9, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
ac270b8
refactor: rework for projections scenario and fixture for public deta…
Nov 26, 2024
dbad0ab
refactor: rework admin fixture and test classes
Nov 26, 2024
64542a2
refactor: format and cleanup
Nov 27, 2024
037d672
fix: or-2582 incorrect read location for historiek
Nov 27, 2024
b88bbaf
refactor: or-2582 rename folders for scenario files
Nov 27, 2024
84e1e66
refactor: or-2582 use base class for fixture setup
Nov 27, 2024
3634405
refactor: or-2582 use common class fixture for beheer detail, histori…
Nov 27, 2024
3d3a7cb
refactor: or-2582 use common class fixture for publiek detail and zoeken
Nov 27, 2024
e35b7c6
refactor: or-2582 use host context instead of direct alba host
Nov 27, 2024
b43a5a3
fix: or-2582 incorrect document session retrieval caused out-of-sync
Nov 27, 2024
aa91f61
fix: or-2582 temporary fix for the marten time gapping with projections
Nov 27, 2024
5d682e7
fix: or-2582 add delay when getting results inside projection tests
Nov 28, 2024
1d66209
fix: or-2582 use seperate appsettings for beheer and publiek hosts
Nov 28, 2024
4dfc797
fix: or-2582 use seperate appsettings for beheer and publiek hosts
Nov 28, 2024
fac04e9
WIP: or-2582 use right way of projectiontests
emalfroy Dec 6, 2024
86a6ba3
test: or-2582 use public store for public projection tests
emalfroy Dec 9, 2024
66dd846
chore: or-2582 use correct scenario
koenmetsu Dec 9, 2024
a54665d
test: or-2582 set up elastic client
koenmetsu Dec 9, 2024
3fb555b
test: or-2582 add support for public projection rebuild in projection…
Dec 9, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ private static IServiceCollection AddMappingsForVerenigingZoek(this IServiceColl
private static GetMappingResponse GetMappingFor<T>(this IServiceProvider serviceProvider) where T : class
=> serviceProvider.GetRequiredService<ElasticClient>().Indices.GetMapping<T>();

private static ElasticClient CreateElasticClient(ElasticSearchOptionsSection elasticSearchOptions, ILogger logger)
public static ElasticClient CreateElasticClient(ElasticSearchOptionsSection elasticSearchOptions, ILogger logger)
{
var settings = new ConnectionSettings(new Uri(elasticSearchOptions.Uri!))
.BasicAuthentication(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,27 @@ public static IServiceCollection ConfigureProjectionsWithMarten(
private static MartenServiceCollectionExtensions.MartenConfigurationExpression AddMarten(
IServiceCollection services,
ConfigurationManager configurationManager)
{
var martenConfigurationExpression = services.AddMarten(
serviceProvider =>
{
var opts = new StoreOptions();

return ConfigureStoreOptions(opts, serviceProvider.GetRequiredService<ILogger<LocatieLookupProjection>>(), serviceProvider.GetRequiredService<ILogger<LocatieZonderAdresMatchProjection>>(), serviceProvider.GetRequiredService<IElasticRepository>(), serviceProvider.GetRequiredService<IHostEnvironment>().IsDevelopment(), serviceProvider.GetRequiredService<ILogger<BeheerZoekenEventsConsumer>>(), configurationManager.GetSection(PostgreSqlOptionsSection.SectionName)
.Get<PostgreSqlOptionsSection>());
});

return martenConfigurationExpression;
}

public static StoreOptions ConfigureStoreOptions(
StoreOptions opts,
ILogger<LocatieLookupProjection> locatieLookupLogger,
ILogger<LocatieZonderAdresMatchProjection> locatieZonderAdresMatchProjectionLogger,
IElasticRepository elasticRepository,
bool isDevelopment,
ILogger<BeheerZoekenEventsConsumer> beheerZoekenEventsConsumerLogger,
PostgreSqlOptionsSection? postgreSqlOptionsSection)
{
static string GetPostgresConnectionString(PostgreSqlOptionsSection postgreSqlOptions)
=> $"host={postgreSqlOptions.Host};" +
Expand All @@ -70,97 +91,89 @@ static JsonNetSerializer CreateCustomMartenSerializer()
return jsonNetSerializer;
}

var martenConfigurationExpression = services.AddMarten(
serviceProvider =>
{
var postgreSqlOptions = configurationManager.GetSection(PostgreSqlOptionsSection.SectionName)
.Get<PostgreSqlOptionsSection>() ??
throw new ConfigurationErrorsException("Missing a valid postgres configuration");
var postgreSqlOptions = postgreSqlOptionsSection ??
throw new ConfigurationErrorsException("Missing a valid postgres configuration");

var connectionString = GetPostgresConnectionString(postgreSqlOptions);
var connectionString = GetPostgresConnectionString(postgreSqlOptions);

var opts = new StoreOptions();

if (!string.IsNullOrEmpty(postgreSqlOptions.Schema))
{
opts.Events.DatabaseSchemaName = postgreSqlOptions.Schema;
opts.DatabaseSchemaName = postgreSqlOptions.Schema;
}
if (!string.IsNullOrEmpty(postgreSqlOptions.Schema))
{
opts.Events.DatabaseSchemaName = postgreSqlOptions.Schema;
opts.DatabaseSchemaName = postgreSqlOptions.Schema;
}

opts.Connection(connectionString);
opts.Connection(connectionString);

opts.OpenTelemetry.TrackConnections = TrackLevel.Normal;
opts.OpenTelemetry.TrackEventCounters();

opts.Events.StreamIdentity = StreamIdentity.AsString;

opts.Events.MetadataConfig.EnableAll();
opts.Events.MetadataConfig.EnableAll();

opts.Projections.StaleSequenceThreshold = TimeSpan.FromSeconds(30);
opts.Projections.StaleSequenceThreshold = TimeSpan.FromSeconds(30);

opts.Projections.DaemonLockId = 1;
opts.Projections.DaemonLockId = 1;

opts.Projections.Add(new BeheerVerenigingHistoriekProjection(), ProjectionLifecycle.Async);
opts.Projections.Add(new BeheerVerenigingDetailProjection(), ProjectionLifecycle.Async);
opts.Projections.Add(new PowerBiExportProjection(), ProjectionLifecycle.Async);
opts.Projections.Add(new BeheerKboSyncHistoriekProjection(), ProjectionLifecycle.Async);
opts.Projections.Add( new LocatieLookupProjection(serviceProvider.GetRequiredService<ILogger<LocatieLookupProjection>>()), ProjectionLifecycle.Async);
opts.Projections.Add( new LocatieZonderAdresMatchProjection(serviceProvider.GetRequiredService<ILogger<LocatieZonderAdresMatchProjection>>()), ProjectionLifecycle.Async);
opts.Projections.Add(new BeheerVerenigingHistoriekProjection(), ProjectionLifecycle.Async);
opts.Projections.Add(new BeheerVerenigingDetailProjection(), ProjectionLifecycle.Async);
opts.Projections.Add(new PowerBiExportProjection(), ProjectionLifecycle.Async);
opts.Projections.Add(new BeheerKboSyncHistoriekProjection(), ProjectionLifecycle.Async);
opts.Projections.Add( new LocatieLookupProjection(locatieLookupLogger), ProjectionLifecycle.Async);
opts.Projections.Add( new LocatieZonderAdresMatchProjection(locatieZonderAdresMatchProjectionLogger), ProjectionLifecycle.Async);

opts.Projections.Add(
new MartenSubscription(
new BeheerZoekenEventsConsumer(
new BeheerZoekProjectionHandler(
serviceProvider.GetRequiredService<IElasticRepository>()
),
serviceProvider.GetRequiredService<ILogger<BeheerZoekenEventsConsumer>>())
opts.Projections.Add(
new MartenSubscription(
new BeheerZoekenEventsConsumer(
new BeheerZoekProjectionHandler(
elasticRepository
),
ProjectionLifecycle.Async,
ProjectionNames.BeheerZoek);

opts.Projections.Add(
new MartenSubscription(
new DuplicateDetectionEventsConsumer(
new DuplicateDetectionProjectionHandler(
serviceProvider.GetRequiredService<IElasticRepository>())
)
),
ProjectionLifecycle.Async,
ProjectionNames.DuplicateDetection);

opts.Serializer(CreateCustomMartenSerializer());

opts.RegisterDocumentType<BeheerVerenigingDetailDocument>();
opts.RegisterDocumentType<PowerBiExportDocument>();
opts.RegisterDocumentType<BeheerVerenigingHistoriekDocument>();
opts.RegisterDocumentType<BeheerKboSyncHistoriekGebeurtenisDocument>();
opts.RegisterDocumentType<LocatieLookupDocument>();
opts.RegisterDocumentType<LocatieZonderAdresMatchDocument>();

opts.Schema.For<LocatieLookupDocument>()
.UseNumericRevisions(true)
.UseOptimisticConcurrency(false);
opts.Schema.For<LocatieZonderAdresMatchDocument>()
.UseNumericRevisions(true)
.UseOptimisticConcurrency(false);

opts.Schema.For<PowerBiExportDocument>()
.UseNumericRevisions(true)
.UseOptimisticConcurrency(false);

if (serviceProvider.GetRequiredService<IHostEnvironment>().IsDevelopment())
{
opts.GeneratedCodeMode = TypeLoadMode.Dynamic;
}
else
{
opts.GeneratedCodeMode = TypeLoadMode.Static;
opts.SourceCodeWritingEnabled = false;
}

return opts;
});
beheerZoekenEventsConsumerLogger)
),
ProjectionLifecycle.Async,
ProjectionNames.BeheerZoek);

opts.Projections.Add(
new MartenSubscription(
new DuplicateDetectionEventsConsumer(
new DuplicateDetectionProjectionHandler(
elasticRepository)
)
),
ProjectionLifecycle.Async,
ProjectionNames.DuplicateDetection);

opts.Serializer(CreateCustomMartenSerializer());

opts.RegisterDocumentType<BeheerVerenigingDetailDocument>();
opts.RegisterDocumentType<PowerBiExportDocument>();
opts.RegisterDocumentType<BeheerVerenigingHistoriekDocument>();
opts.RegisterDocumentType<BeheerKboSyncHistoriekGebeurtenisDocument>();
opts.RegisterDocumentType<LocatieLookupDocument>();
opts.RegisterDocumentType<LocatieZonderAdresMatchDocument>();

opts.Schema.For<LocatieLookupDocument>()
.UseNumericRevisions(true)
.UseOptimisticConcurrency(false);
opts.Schema.For<LocatieZonderAdresMatchDocument>()
.UseNumericRevisions(true)
.UseOptimisticConcurrency(false);

opts.Schema.For<PowerBiExportDocument>()
.UseNumericRevisions(true)
.UseOptimisticConcurrency(false);

if (isDevelopment)
{
opts.GeneratedCodeMode = TypeLoadMode.Dynamic;
}
else
{
opts.GeneratedCodeMode = TypeLoadMode.Static;
opts.SourceCodeWritingEnabled = false;
}

return martenConfigurationExpression;
return opts;
}
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
namespace AssociationRegistry.Public.Api.Infrastructure.Extensions;

using ConfigurationBindings;
using Framework;
using Hosts.Configuration.ConfigurationBindings;
using Microsoft.Extensions.Configuration;
using System;

Expand All @@ -10,8 +10,8 @@ public static class ConfigurationExtensions
public static AssociationRegistry.Hosts.Configuration.ConfigurationBindings.PostgreSqlOptionsSection GetPostgreSqlOptionsSection(this IConfiguration configuration)
{
var postgreSqlOptionsSection = configuration
.GetSection(PostgreSqlOptionsSection.Name)
.Get<AssociationRegistry.Hosts.Configuration.ConfigurationBindings.PostgreSqlOptionsSection>();
.GetSection(PostgreSqlOptionsSection.SectionName)
.Get<PostgreSqlOptionsSection>();

postgreSqlOptionsSection.ThrowIfInvalid();

Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,9 @@
namespace AssociationRegistry.Public.Api.Infrastructure.Extensions;

using ConfigurationBindings;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Identity.Client;
using Hosts.Configuration.ConfigurationBindings;
using Nest;
using Schema;
using Schema.Search;
using System;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Text;
Expand All @@ -18,8 +14,13 @@ public static IServiceCollection AddElasticSearch(
this IServiceCollection services,
ElasticSearchOptionsSection elasticSearchOptions)
{
services.AddSingleton<ElasticClient>(serviceProvider => CreateElasticClient(elasticSearchOptions, serviceProvider.GetRequiredService<ILogger<ElasticClient>>()));
services.AddSingleton<IElasticClient>(serviceProvider => CreateElasticClient(elasticSearchOptions, serviceProvider.GetRequiredService<ILogger<ElasticClient>>()));
services.AddSingleton<ElasticClient>(serviceProvider
=> CreateElasticClient(elasticSearchOptions,
serviceProvider.GetRequiredService<ILogger<ElasticClient>>()));

services.AddSingleton<IElasticClient>(serviceProvider
=> CreateElasticClient(elasticSearchOptions,
serviceProvider.GetRequiredService<ILogger<ElasticClient>>()));

services.AddSingleton(serviceProvider =>
{
Expand All @@ -32,28 +33,28 @@ public static IServiceCollection AddElasticSearch(
return services;
}

private static ElasticClient CreateElasticClient(ElasticSearchOptionsSection elasticSearchOptions, ILogger logger)
public static ElasticClient CreateElasticClient(ElasticSearchOptionsSection elasticSearchOptions, ILogger logger)
{
var settings = new ConnectionSettings(
new Uri(elasticSearchOptions.Uri!))
.BasicAuthentication(
elasticSearchOptions.Username,
elasticSearchOptions.Password)
.ServerCertificateValidationCallback((o, certificate, arg3, arg4) =>
{
logger.LogWarning("Policy errors: [{Cert}|{Chain}] {Error}", certificate.Subject, arg3, arg4.ToString());
logger.LogWarning("Policy object: {@Error}", o);
.ServerCertificateValidationCallback((o, certificate, arg3, arg4) =>
{
logger.LogWarning("Policy errors: [{Cert}|{Chain}] {Error}", certificate.Subject, arg3, arg4.ToString());
logger.LogWarning("Policy object: {@Error}", o);

if (arg4 != SslPolicyErrors.None)
{
logger.LogWarning(Convert.ToBase64String(certificate.Export(X509ContentType.Cert),
Base64FormattingOptions.InsertLineBreaks));
if (arg4 != SslPolicyErrors.None)
{
logger.LogWarning(Convert.ToBase64String(certificate.Export(X509ContentType.Cert),
Base64FormattingOptions.InsertLineBreaks));

LogCertificateDetails(certificate, arg3);
}
LogCertificateDetails(certificate, arg3);
}

return true;
})
return true;
})
.IncludeServerStackTraceOnError()
.MapVerenigingDocument(elasticSearchOptions.Indices!.Verenigingen!);

Expand Down
Loading
Loading