From 20775c5da0bc8eea4f689bf7447b52c8b44c56b8 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Wed, 14 Aug 2024 13:26:36 -0400 Subject: [PATCH] Add support for API diagnostics (addresses #616) --- tools/code/common.tests/Api.cs | 9 +- tools/code/common.tests/ApiDiagnostic.cs | 86 ++++ tools/code/common.tests/Service.cs | 6 +- tools/code/common/ApiDiagnostic.cs | 306 ++++++++++++ tools/code/common/common.csproj | 8 +- tools/code/extractor/Api.cs | 3 + tools/code/extractor/ApiDiagnostic.cs | 104 +++++ tools/code/extractor/extractor.csproj | 2 +- tools/code/integration.tests/Api.cs | 40 +- tools/code/integration.tests/ApiDiagnostic.cs | 435 ++++++++++++++++++ tools/code/integration.tests/Test.cs | 2 +- tools/code/publisher/ApiDiagnostic.cs | 249 ++++++++++ tools/code/publisher/App.cs | 6 + tools/code/publisher/publisher.csproj | 36 +- 14 files changed, 1258 insertions(+), 34 deletions(-) create mode 100644 tools/code/common.tests/ApiDiagnostic.cs create mode 100644 tools/code/common/ApiDiagnostic.cs create mode 100644 tools/code/extractor/ApiDiagnostic.cs create mode 100644 tools/code/integration.tests/ApiDiagnostic.cs create mode 100644 tools/code/publisher/ApiDiagnostic.cs diff --git a/tools/code/common.tests/Api.cs b/tools/code/common.tests/Api.cs index bd66b205..fb3e7767 100644 --- a/tools/code/common.tests/Api.cs +++ b/tools/code/common.tests/Api.cs @@ -16,8 +16,8 @@ public sealed record Soap : ApiType; public static Gen Generate() => Gen.OneOfConst(new GraphQl(), new Http()); - //new WebSocket(), - //new Soap()); + //new WebSocket(), + //new Soap()); } public record ApiRevision @@ -106,6 +106,7 @@ public record ApiModel public required ApiType Type { get; init; } public required string Path { get; init; } public required FrozenSet Revisions { get; init; } + public required FrozenSet Diagnostics { get; init; } public Option Version { get; init; } = Option.None; public static Gen Generate() => @@ -113,12 +114,14 @@ from type in ApiType.Generate() from name in GenerateName(type) from path in GeneratePath() from revisions in ApiRevision.GenerateSet(type, name) + from diagnostics in ApiDiagnosticModel.GenerateSet() select new ApiModel { Name = name, Type = type, Path = path, - Revisions = revisions + Revisions = revisions, + Diagnostics = diagnostics }; public static Gen GenerateName(ApiType type) => diff --git a/tools/code/common.tests/ApiDiagnostic.cs b/tools/code/common.tests/ApiDiagnostic.cs new file mode 100644 index 00000000..4ca2482e --- /dev/null +++ b/tools/code/common.tests/ApiDiagnostic.cs @@ -0,0 +1,86 @@ +using CsCheck; +using LanguageExt; +using Microsoft.Extensions.Logging; +using Microsoft.OpenApi.Readers.Interface; +using Nito.Comparers; +using System.Collections; +using System.Collections.Frozen; +using System.Collections.Generic; +using System.Linq; + +namespace common.tests; + +public sealed record ApiDiagnosticSampling +{ + public required string Type { get; init; } + public required float Percentage { get; init; } + + public static Gen Generate() => + from type in Gen.Const("fixed") + from percentage in Gen.Int[0, 100] + select new ApiDiagnosticSampling + { + Type = type, + Percentage = percentage + }; +} + +public record ApiDiagnosticModel +{ + public required ApiDiagnosticName Name { get; init; } + public required LoggerName LoggerName { get; init; } + public Option AlwaysLog { get; init; } + public Option Sampling { get; init; } + + public static Gen Generate() => + from loggerType in LoggerType.Generate() + from loggerName in LoggerModel.GenerateName(loggerType) + from name in GenerateName(loggerType) + from alwaysLog in Gen.Const("allErrors").OptionOf() + from sampling in ApiDiagnosticSampling.Generate().OptionOf() + select new ApiDiagnosticModel + { + Name = name, + LoggerName = loggerName, + AlwaysLog = alwaysLog, + Sampling = sampling + }; + + public static Gen GenerateName() => + from name in Generator.AlphaNumericStringBetween(10, 20) + select ApiDiagnosticName.From(name); + + private static Gen GenerateName(LoggerType loggerType) => + loggerType switch + { + LoggerType.AzureMonitor => Gen.Const(ApiDiagnosticName.From("azuremonitor")), + _ => GenerateName() + }; + + public static Gen> GenerateSet() => + Generate().FrozenSetOf(0, 20, Comparer); + + public static Gen> GenerateSet(FrozenSet originalSet, ICollection loggers) + { + if (loggers.Count == 0) + { + Gen.Const(Enumerable.Empty().ToFrozenSet(Comparer)); + } + + var loggersArray = loggers.ToArray(); + + return from updates in originalSet.Select(diagnostic => from logger in Gen.OneOfConst(loggersArray) + select diagnostic with + { + // Diagnostic name must be "azuremonitor" if the logger type is AzureMonitor + Name = logger.Type is LoggerType.AzureMonitor ? ApiDiagnosticName.From("azuremonitor") : diagnostic.Name, + LoggerName = logger.Name + }) + .SequenceToFrozenSet(originalSet.Comparer) + select updates; + } + + private static IEqualityComparer Comparer { get; } = + EqualityComparerBuilder.For() + .EquateBy(x => x.Name); +} diff --git a/tools/code/common.tests/Service.cs b/tools/code/common.tests/Service.cs index bb1ec6fd..4bfcdf44 100644 --- a/tools/code/common.tests/Service.cs +++ b/tools/code/common.tests/Service.cs @@ -36,7 +36,7 @@ select updatedDiagnostics from policyFragments in PolicyFragmentModel.GenerateSet() from servicePolicies in ServicePolicyModel.GenerateSet() from apis in from originalApis in ApiModel.GenerateSet() - from updatedApis in UpdateApis(originalApis, versionSets, tags) + from updatedApis in UpdateApis(originalApis, versionSets, tags, loggers) select updatedApis from groups in GroupModel.GenerateSet() from products in from originalProducts in ProductModel.GenerateSet() @@ -67,9 +67,11 @@ select updatedSubscriptions public static Gen> UpdateApis(FrozenSet apis, ICollection versionSets, - ICollection tags) => + ICollection tags, + ICollection loggers) => apis.Select(api => from version in UpdateApiVersion(versionSets) from revisions in UpdateApiRevisions(api.Revisions, tags) + from diagnostics in ApiDiagnosticModel.GenerateSet(api.Diagnostics, loggers) select api with { Version = version, diff --git a/tools/code/common/ApiDiagnostic.cs b/tools/code/common/ApiDiagnostic.cs new file mode 100644 index 00000000..bfe78193 --- /dev/null +++ b/tools/code/common/ApiDiagnostic.cs @@ -0,0 +1,306 @@ +using Azure.Core.Pipeline; +using Flurl; +using LanguageExt; +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Linq; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; + +namespace common; + +public sealed record ApiDiagnosticName : ResourceName, IResourceName +{ + private ApiDiagnosticName(string value) : base(value) { } + + public static ApiDiagnosticName From(string value) => new(value); +} + +public sealed record ApiDiagnosticsUri : ResourceUri +{ + public required ApiUri Parent { get; init; } + + private static string PathSegment { get; } = "diagnostics"; + + protected override Uri Value => + Parent.ToUri().AppendPathSegment(PathSegment).ToUri(); + + public static ApiDiagnosticsUri From(ApiName apiName, ManagementServiceUri serviceUri) => + new() { Parent = ApiUri.From(apiName, serviceUri) }; +} + +public sealed record ApiDiagnosticUri : ResourceUri +{ + public required ApiDiagnosticsUri Parent { get; init; } + + public required ApiDiagnosticName Name { get; init; } + + protected override Uri Value => + Parent.ToUri().AppendPathSegment(Name.ToString()).ToUri(); + + public static ApiDiagnosticUri From(ApiDiagnosticName name, ApiName apiName, ManagementServiceUri serviceUri) => + new() + { + Parent = ApiDiagnosticsUri.From(apiName, serviceUri), + Name = name + }; +} + +public sealed record ApiDiagnosticsDirectory : ResourceDirectory +{ + public required ApiDirectory Parent { get; init; } + + private static string Name { get; } = "diagnostics"; + + protected override DirectoryInfo Value => + Parent.ToDirectoryInfo().GetChildDirectory(Name); + + public static ApiDiagnosticsDirectory From(ApiName apiName, ManagementServiceDirectory serviceDirectory) => + new() { Parent = ApiDirectory.From(apiName, serviceDirectory) }; + + public static Option TryParse(DirectoryInfo? directory, ManagementServiceDirectory serviceDirectory) => + directory?.Name == Name + ? from parent in ApiDirectory.TryParse(directory.Parent, serviceDirectory) + select new ApiDiagnosticsDirectory { Parent = parent } + : Option.None; +} + +public sealed record ApiDiagnosticDirectory : ResourceDirectory +{ + public required ApiDiagnosticsDirectory Parent { get; init; } + + public required ApiDiagnosticName Name { get; init; } + + protected override DirectoryInfo Value => + Parent.ToDirectoryInfo().GetChildDirectory(Name.Value); + + public static ApiDiagnosticDirectory From(ApiDiagnosticName name, ApiName apiName, ManagementServiceDirectory serviceDirectory) => + new() + { + Parent = ApiDiagnosticsDirectory.From(apiName, serviceDirectory), + Name = name + }; + + public static Option TryParse(DirectoryInfo? directory, ManagementServiceDirectory serviceDirectory) => + from parent in ApiDiagnosticsDirectory.TryParse(directory?.Parent, serviceDirectory) + let name = ApiDiagnosticName.From(directory!.Name) + select new ApiDiagnosticDirectory + { + Parent = parent, + Name = name + }; +} + +public sealed record ApiDiagnosticInformationFile : ResourceFile +{ + public required ApiDiagnosticDirectory Parent { get; init; } + + private static string Name { get; } = "diagnosticInformation.json"; + + protected override FileInfo Value => + Parent.ToDirectoryInfo().GetChildFile(Name); + + public static ApiDiagnosticInformationFile From(ApiDiagnosticName name, ApiName apiName, ManagementServiceDirectory serviceDirectory) => + new() + { + Parent = ApiDiagnosticDirectory.From(name, apiName, serviceDirectory) + }; + + public static Option TryParse(FileInfo? file, ManagementServiceDirectory serviceDirectory) => + file?.Name == Name + ? from parent in ApiDiagnosticDirectory.TryParse(file.Directory, serviceDirectory) + select new ApiDiagnosticInformationFile + { + Parent = parent + } + : Option.None; +} + +public sealed record ApiDiagnosticDto +{ + [JsonPropertyName("properties")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public required DiagnosticContract Properties { get; init; } + + public sealed record DiagnosticContract + { + [JsonPropertyName("loggerId")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public string? LoggerId { get; init; } + + [JsonPropertyName("alwaysLog")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public string? AlwaysLog { get; init; } + + [JsonPropertyName("backend")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public PipelineDiagnosticSettings? Backend { get; init; } + + [JsonPropertyName("frontend")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public PipelineDiagnosticSettings? Frontend { get; init; } + + [JsonPropertyName("httpCorrelationProtocol")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public string? HttpCorrelationProtocol { get; init; } + + [JsonPropertyName("logClientIp")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public bool? LogClientIp { get; init; } + + [JsonPropertyName("metrics")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public bool? Metrics { get; init; } + + [JsonPropertyName("operationNameFormat")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public string? OperationNameFormat { get; init; } + + [JsonPropertyName("sampling")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public SamplingSettings? Sampling { get; init; } + + [JsonPropertyName("verbosity")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public string? Verbosity { get; init; } + } + + public sealed record PipelineDiagnosticSettings + { + [JsonPropertyName("request")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public HttpMessageDiagnostic? Request { get; init; } + + [JsonPropertyName("response")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public HttpMessageDiagnostic? Response { get; init; } + } + + public sealed record HttpMessageDiagnostic + { + [JsonPropertyName("body")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public BodyDiagnosticSettings? Body { get; init; } + + [JsonPropertyName("dataMasking")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public DataMasking? DataMasking { get; init; } + + [JsonPropertyName("headers")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray? Headers { get; init; } + } + + public sealed record BodyDiagnosticSettings + { + [JsonPropertyName("bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public int? Bytes { get; init; } + } + + public sealed record DataMasking + { + [JsonPropertyName("headers")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray? Headers { get; init; } + + [JsonPropertyName("queryParams")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray? QueryParams { get; init; } + } + + public sealed record DataMaskingEntity + { + [JsonPropertyName("mode")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public string? Mode { get; init; } + + [JsonPropertyName("value")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public string? Value { get; init; } + } + + public sealed record SamplingSettings + { + [JsonPropertyName("percentage")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public float? Percentage { get; init; } + + [JsonPropertyName("samplingType")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public string? SamplingType { get; init; } + } +} + +public static class ApiDiagnosticModule +{ + public static async ValueTask DeleteAll(this ApiDiagnosticsUri uri, HttpPipeline pipeline, CancellationToken cancellationToken) => + await uri.ListNames(pipeline, cancellationToken) + .IterParallel(async name => + { + var resourceUri = new ApiDiagnosticUri { Parent = uri, Name = name }; + await resourceUri.Delete(pipeline, cancellationToken); + }, cancellationToken); + + public static IAsyncEnumerable ListNames(this ApiDiagnosticsUri uri, HttpPipeline pipeline, CancellationToken cancellationToken) => + pipeline.ListJsonObjects(uri.ToUri(), cancellationToken) + .Select(jsonObject => jsonObject.GetStringProperty("name")) + .Select(ApiDiagnosticName.From); + + public static IAsyncEnumerable<(ApiDiagnosticName Name, ApiDiagnosticDto Dto)> List(this ApiDiagnosticsUri uri, HttpPipeline pipeline, CancellationToken cancellationToken) => + uri.ListNames(pipeline, cancellationToken) + .SelectAwait(async name => + { + var resourceUri = new ApiDiagnosticUri { Parent = uri, Name = name }; + var dto = await resourceUri.GetDto(pipeline, cancellationToken); + return (name, dto); + }); + + public static async ValueTask GetDto(this ApiDiagnosticUri uri, HttpPipeline pipeline, CancellationToken cancellationToken) + { + var content = await pipeline.GetContent(uri.ToUri(), cancellationToken); + return content.ToObjectFromJson(); + } + + public static async ValueTask Delete(this ApiDiagnosticUri uri, HttpPipeline pipeline, CancellationToken cancellationToken) => + await pipeline.DeleteResource(uri.ToUri(), waitForCompletion: true, cancellationToken); + + public static async ValueTask PutDto(this ApiDiagnosticUri uri, ApiDiagnosticDto dto, HttpPipeline pipeline, CancellationToken cancellationToken) + { + var content = BinaryData.FromObjectAsJson(dto); + await pipeline.PutContent(uri.ToUri(), content, cancellationToken); + } + + public static IEnumerable ListDirectories(ManagementServiceDirectory serviceDirectory) => + from apiDirectory in ApiModule.ListDirectories(serviceDirectory) + let diagnosticsDirectory = new ApiDiagnosticsDirectory { Parent = apiDirectory } + where diagnosticsDirectory.ToDirectoryInfo().Exists() + from diagnosticDirectoryInfo in diagnosticsDirectory.ToDirectoryInfo().ListDirectories("*") + let name = ApiDiagnosticName.From(diagnosticDirectoryInfo.Name) + select new ApiDiagnosticDirectory + { + Parent = diagnosticsDirectory, + Name = name + }; + + public static IEnumerable ListInformationFiles(ManagementServiceDirectory serviceDirectory) => + from diagnosticDirectory in ListDirectories(serviceDirectory) + let informationFile = new ApiDiagnosticInformationFile { Parent = diagnosticDirectory } + where informationFile.ToFileInfo().Exists() + select informationFile; + + public static async ValueTask WriteDto(this ApiDiagnosticInformationFile file, ApiDiagnosticDto dto, CancellationToken cancellationToken) + { + var content = BinaryData.FromObjectAsJson(dto, JsonObjectExtensions.SerializerOptions); + await file.ToFileInfo().OverwriteWithBinaryData(content, cancellationToken); + } + + public static async ValueTask ReadDto(this ApiDiagnosticInformationFile file, CancellationToken cancellationToken) + { + var content = await file.ToFileInfo().ReadAsBinaryData(cancellationToken); + return content.ToObjectFromJson(); + } +} diff --git a/tools/code/common/common.csproj b/tools/code/common/common.csproj index 62768daa..73408eb4 100644 --- a/tools/code/common/common.csproj +++ b/tools/code/common/common.csproj @@ -10,7 +10,7 @@ - + @@ -18,10 +18,10 @@ - + - + @@ -32,7 +32,7 @@ - + diff --git a/tools/code/extractor/Api.cs b/tools/code/extractor/Api.cs index 2b579401..584c27fe 100644 --- a/tools/code/extractor/Api.cs +++ b/tools/code/extractor/Api.cs @@ -29,6 +29,7 @@ public static void ConfigureExtractApis(IHostApplicationBuilder builder) ConfigureWriteApiArtifacts(builder); ApiPolicyModule.ConfigureExtractApiPolicies(builder); ApiTagModule.ConfigureExtractApiTags(builder); + ApiDiagnosticModule.ConfigureExtractApiDiagnostics(builder); ApiOperationModule.ConfigureExtractApiOperations(builder); builder.Services.TryAddSingleton(GetExtractApis); @@ -40,6 +41,7 @@ private static ExtractApis GetExtractApis(IServiceProvider provider) var writeArtifacts = provider.GetRequiredService(); var extractApiPolicies = provider.GetRequiredService(); var extractApiTags = provider.GetRequiredService(); + var extractApiDiagnostics = provider.GetRequiredService(); var extractApiOperations = provider.GetRequiredService(); var activitySource = provider.GetRequiredService(); var logger = provider.GetRequiredService(); @@ -64,6 +66,7 @@ async ValueTask extractApi(ApiName name, ApiDto dto, Option<(ApiSpecification Sp await writeArtifacts(name, dto, specificationOption, cancellationToken); await extractApiPolicies(name, cancellationToken); await extractApiTags(name, cancellationToken); + await extractApiDiagnostics(name, cancellationToken); await extractApiOperations(name, cancellationToken); } } diff --git a/tools/code/extractor/ApiDiagnostic.cs b/tools/code/extractor/ApiDiagnostic.cs new file mode 100644 index 00000000..97f51adf --- /dev/null +++ b/tools/code/extractor/ApiDiagnostic.cs @@ -0,0 +1,104 @@ +using Azure.Core.Pipeline; +using common; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; + +namespace extractor; + +public delegate ValueTask ExtractApiDiagnostics(ApiName apiName, CancellationToken cancellationToken); +public delegate IAsyncEnumerable<(ApiDiagnosticName Name, ApiDiagnosticDto Dto)> ListApiDiagnostics(ApiName apiName, CancellationToken cancellationToken); +public delegate ValueTask WriteApiDiagnosticArtifacts(ApiDiagnosticName name, ApiDiagnosticDto dto, ApiName apiName, CancellationToken cancellationToken); +public delegate ValueTask WriteApiDiagnosticInformationFile(ApiDiagnosticName name, ApiDiagnosticDto dto, ApiName apiName, CancellationToken cancellationToken); + +internal static class ApiDiagnosticModule +{ + public static void ConfigureExtractApiDiagnostics(IHostApplicationBuilder builder) + { + ConfigureListApiDiagnostics(builder); + ConfigureWriteApiDiagnosticArtifacts(builder); + + builder.Services.TryAddSingleton(GetExtractApiDiagnostics); + } + + private static ExtractApiDiagnostics GetExtractApiDiagnostics(IServiceProvider provider) + { + var list = provider.GetRequiredService(); + var writeArtifacts = provider.GetRequiredService(); + var activitySource = provider.GetRequiredService(); + var logger = provider.GetRequiredService(); + + return async (apiName, cancellationToken) => + { + using var _ = activitySource.StartActivity(nameof(ExtractApiDiagnostics)); + + logger.LogInformation("Extracting diagnostics for API {ApiName}...", apiName); + + await list(apiName, cancellationToken) + .IterParallel(async resource => await writeArtifacts(resource.Name, resource.Dto, apiName, cancellationToken), + cancellationToken); + }; + } + + private static void ConfigureListApiDiagnostics(IHostApplicationBuilder builder) + { + AzureModule.ConfigureManagementServiceUri(builder); + AzureModule.ConfigureHttpPipeline(builder); + + builder.Services.TryAddSingleton(GetListApiDiagnostics); + } + + private static ListApiDiagnostics GetListApiDiagnostics(IServiceProvider provider) + { + var serviceUri = provider.GetRequiredService(); + var pipeline = provider.GetRequiredService(); + + return (apiName, cancellationToken) => + { + var diagnosticsUri = ApiDiagnosticsUri.From(apiName, serviceUri); + return diagnosticsUri.List(pipeline, cancellationToken); + }; + } + + private static void ConfigureWriteApiDiagnosticArtifacts(IHostApplicationBuilder builder) + { + ConfigureWriteApiDiagnosticInformationFile(builder); + + builder.Services.TryAddSingleton(GetWriteApiDiagnosticArtifacts); + } + + private static WriteApiDiagnosticArtifacts GetWriteApiDiagnosticArtifacts(IServiceProvider provider) + { + var writeInformationFile = provider.GetRequiredService(); + + return async (name, dto, apiName, cancellationToken) => + await writeInformationFile(name, dto, apiName, cancellationToken); + } + + private static void ConfigureWriteApiDiagnosticInformationFile(IHostApplicationBuilder builder) + { + AzureModule.ConfigureManagementServiceDirectory(builder); + + builder.Services.TryAddSingleton(GetWriteApiDiagnosticInformationFile); + } + + private static WriteApiDiagnosticInformationFile GetWriteApiDiagnosticInformationFile(IServiceProvider provider) + { + var serviceDirectory = provider.GetRequiredService(); + var logger = provider.GetRequiredService(); + + return async (name, dto, apiName, cancellationToken) => + { + var informationFile = ApiDiagnosticInformationFile.From(name, apiName, serviceDirectory); + + logger.LogInformation("Writing API diagnostic information file {ApiDiagnosticInformationFile}...", informationFile); + await informationFile.WriteDto(dto, cancellationToken); + }; + } +} \ No newline at end of file diff --git a/tools/code/extractor/extractor.csproj b/tools/code/extractor/extractor.csproj index 1fc233b0..da427dc9 100644 --- a/tools/code/extractor/extractor.csproj +++ b/tools/code/extractor/extractor.csproj @@ -8,7 +8,7 @@ 6ce9dc18-ea0d-4d82-8f1a-e63877f35b88 Exe enable - 6.0.1 + 6.0.1.1 diff --git a/tools/code/integration.tests/Api.cs b/tools/code/integration.tests/Api.cs index 8f92c345..f8508abf 100644 --- a/tools/code/integration.tests/Api.cs +++ b/tools/code/integration.tests/Api.cs @@ -54,14 +54,15 @@ private static DeleteAllApis GetDeleteAllApis(IServiceProvider provider) logger.LogInformation("Deleting all APIs in {ServiceName}...", serviceName); var serviceUri = getServiceUri(serviceName); + var apisUri = ApisUri.From(serviceUri); - await ApisUri.From(serviceUri) - .DeleteAll(pipeline, cancellationToken); + await apisUri.DeleteAll(pipeline, cancellationToken); }; } public static void ConfigurePutApiModels(IHostApplicationBuilder builder) { + ApiDiagnosticModule.ConfigurePutApiDiagnosticModels(builder); ManagementServiceModule.ConfigureGetManagementServiceUri(builder); AzureModule.ConfigureHttpPipeline(builder); @@ -70,6 +71,7 @@ public static void ConfigurePutApiModels(IHostApplicationBuilder builder) private static PutApiModels GetPutApiModels(IServiceProvider provider) { + var putDiagnostics = provider.GetRequiredService(); var getServiceUri = provider.GetRequiredService(); var pipeline = provider.GetRequiredService(); var activitySource = provider.GetRequiredService(); @@ -83,23 +85,26 @@ private static PutApiModels GetPutApiModels(IServiceProvider provider) await models.IterParallel(async model => { - async ValueTask putRevision(ApiRevision revision) => await put(model.Name, model.Type, model.Path, model.Version, revision, serviceName, cancellationToken); + var serviceUri = getServiceUri(serviceName); + + async ValueTask putRevision(ApiRevision revision) => await put(model.Name, model.Type, model.Path, model.Version, revision, serviceUri, cancellationToken); // Put first revision to make sure it's the current revision. await model.Revisions.HeadOrNone().IterTask(putRevision); // Put other revisions await model.Revisions.Skip(1).IterParallel(putRevision, cancellationToken); + + await putDiagnostics(model.Diagnostics, model.Name, serviceName, cancellationToken); }, cancellationToken); }; - async ValueTask put(ApiName name, ApiType type, string path, Option version, ApiRevision revision, ManagementServiceName serviceName, CancellationToken cancellationToken) + async ValueTask put(ApiName name, ApiType type, string path, Option version, ApiRevision revision, ManagementServiceUri serviceUri, CancellationToken cancellationToken) { var rootName = ApiName.GetRootName(name); var dto = getDto(rootName, type, path, version, revision); var revisionedName = ApiName.GetRevisionedName(rootName, revision.Number); - var serviceUri = getServiceUri(serviceName); var uri = ApiUri.From(revisionedName, serviceUri); await uri.PutDto(dto, pipeline, cancellationToken); @@ -158,6 +163,7 @@ public static void ConfigureValidateExtractedApis(IHostApplicationBuilder builde ConfigureGetApimApis(builder); ConfigureTryGetApimGraphQlSchema(builder); ConfigureGetFileApis(builder); + ApiDiagnosticModule.ConfigureValidateExtractedApiDiagnostics(builder); builder.Services.TryAddSingleton(GetValidateExtractedApis); } @@ -167,6 +173,7 @@ private static ValidateExtractedApis GetValidateExtractedApis(IServiceProvider p var getApimResources = provider.GetRequiredService(); var tryGetApimGraphQlSchema = provider.GetRequiredService(); var getFileResources = provider.GetRequiredService(); + var validateDiagnostics = provider.GetRequiredService(); var activitySource = provider.GetRequiredService(); var logger = provider.GetRequiredService(); @@ -180,6 +187,14 @@ private static ValidateExtractedApis GetValidateExtractedApis(IServiceProvider p await validateExtractedInformationFiles(expected, serviceDirectory, cancellationToken); await validateExtractedSpecificationFiles(expected, defaultApiSpecification, serviceName, serviceDirectory, cancellationToken); + + await expected.WhereKey(name => ApiName.IsNotRevisioned(name)) + .IterParallel(async kvp => + { + var apiName = kvp.Key; + + await validateDiagnostics(apiName, serviceName, serviceDirectory, cancellationToken); + }, cancellationToken); }; async ValueTask> getExpectedResources(Option> apiNamesOption, Option> versionSetNamesOption, ManagementServiceName serviceName, CancellationToken cancellationToken) @@ -379,11 +394,14 @@ await file.ReadDto(cancellationToken))) public static void ConfigureWriteApiModels(IHostApplicationBuilder builder) { + ApiDiagnosticModule.ConfigureWriteApiDiagnosticModels(builder); + builder.Services.TryAddSingleton(GetWriteApiModels); } private static WriteApiModels GetWriteApiModels(IServiceProvider provider) { + var writeDiagnostics = provider.GetRequiredService(); var activitySource = provider.GetRequiredService(); var logger = provider.GetRequiredService(); @@ -396,6 +414,8 @@ private static WriteApiModels GetWriteApiModels(IServiceProvider provider) await models.IterParallel(async model => { await writeRevisionArtifacts(model, serviceDirectory, cancellationToken); + + await writeDiagnostics(model.Diagnostics, model.Name, serviceDirectory, cancellationToken); }, cancellationToken); }; @@ -497,6 +517,7 @@ public static void ConfigureValidatePublishedApis(IHostApplicationBuilder builde { ConfigureGetFileApis(builder); ConfigureGetApimApis(builder); + ApiDiagnosticModule.ConfigureValidatePublishedApiDiagnostics(builder); builder.Services.TryAddSingleton(GetValidatePublishedApis); } @@ -505,6 +526,7 @@ private static ValidatePublishedApis GetValidatePublishedApis(IServiceProvider p { var getFileResources = provider.GetRequiredService(); var getApimResources = provider.GetRequiredService(); + var validateDiagnostics = provider.GetRequiredService(); var activitySource = provider.GetRequiredService(); var logger = provider.GetRequiredService(); @@ -525,6 +547,14 @@ private static ValidatePublishedApis GetValidatePublishedApis(IServiceProvider p .ToFrozenDictionary(); actual.Should().BeEquivalentTo(expected); + + await expected.WhereKey(name => ApiName.IsNotRevisioned(name)) + .IterParallel(async kvp => + { + var apiName = kvp.Key; + + await validateDiagnostics(apiName, commitIdOption, serviceName, serviceDirectory, cancellationToken); + }, cancellationToken); }; static string normalizeDto(ApiDto dto) => diff --git a/tools/code/integration.tests/ApiDiagnostic.cs b/tools/code/integration.tests/ApiDiagnostic.cs new file mode 100644 index 00000000..2c983dfb --- /dev/null +++ b/tools/code/integration.tests/ApiDiagnostic.cs @@ -0,0 +1,435 @@ +using Azure.Core.Pipeline; +using common; +using common.tests; +using CsCheck; +using FluentAssertions; +using LanguageExt; +using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using publisher; +using System; +using System.Collections.Frozen; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace integration.tests; + +public delegate ValueTask PutApiDiagnosticModels(IEnumerable models, ApiName apiName, ManagementServiceName serviceName, CancellationToken cancellationToken); +public delegate ValueTask ValidateExtractedApiDiagnostics(ApiName apiName, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); +public delegate ValueTask> GetApimApiDiagnostics(ApiName apiName, ManagementServiceName serviceName, CancellationToken cancellationToken); +public delegate ValueTask> GetFileApiDiagnostics(ApiName apiName, ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); +public delegate ValueTask WriteApiDiagnosticModels(IEnumerable models, ApiName apiName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); +public delegate ValueTask ValidatePublishedApiDiagnostics(ApiName apiName, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal static class ApiDiagnosticModule +{ + public static void ConfigurePutApiDiagnosticModels(IHostApplicationBuilder builder) + { + ManagementServiceModule.ConfigureGetManagementServiceUri(builder); + AzureModule.ConfigureHttpPipeline(builder); + + builder.Services.TryAddSingleton(GetPutApiDiagnosticModels); + } + + private static PutApiDiagnosticModels GetPutApiDiagnosticModels(IServiceProvider provider) + { + var getServiceUri = provider.GetRequiredService(); + var pipeline = provider.GetRequiredService(); + var activitySource = provider.GetRequiredService(); + var logger = provider.GetRequiredService(); + + return async (models, apiName, serviceName, cancellationToken) => + { + using var _ = activitySource.StartActivity(nameof(PutApiDiagnosticModels)); + + logger.LogInformation("Putting diagnostic models for API {ApiName} in {ServiceName}...", apiName, serviceName); + + await models.IterParallel(async model => + { + await put(model, apiName, serviceName, cancellationToken); + }, cancellationToken); + }; + + async ValueTask put(ApiDiagnosticModel model, ApiName apiName, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var diagnosticUri = ApiDiagnosticUri.From(model.Name, apiName, serviceUri); + var dto = getDto(model); + + await diagnosticUri.PutDto(dto, pipeline, cancellationToken); + } + + static ApiDiagnosticDto getDto(ApiDiagnosticModel model) => + new() + { + Properties = new ApiDiagnosticDto.DiagnosticContract + { + LoggerId = $"/loggers/{model.LoggerName}", + AlwaysLog = model.AlwaysLog.ValueUnsafe(), + Sampling = model.Sampling.Map(sampling => new ApiDiagnosticDto.SamplingSettings + { + SamplingType = sampling.Type, + Percentage = sampling.Percentage + }).ValueUnsafe() + } + }; + } + + public static void ConfigureValidateExtractedApiDiagnostics(IHostApplicationBuilder builder) + { + ConfigureGetApimApiDiagnostics(builder); + ConfigureGetFileApiDiagnostics(builder); + + builder.Services.TryAddSingleton(GetValidateExtractedApiDiagnostics); + } + + private static ValidateExtractedApiDiagnostics GetValidateExtractedApiDiagnostics(IServiceProvider provider) + { + var getApimResources = provider.GetRequiredService(); + var getFileResources = provider.GetRequiredService(); + var activitySource = provider.GetRequiredService(); + var logger = provider.GetRequiredService(); + + return async (apiName, serviceName, serviceDirectory, cancellationToken) => + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedApiDiagnostics)); + + logger.LogInformation("Validating extracted diagnostic models for API {ApiName} in {ServiceName}...", apiName, serviceName); + + var apimResources = await getApimResources(apiName, serviceName, cancellationToken); + var fileResources = await getFileResources(apiName, serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.MapValue(normalizeDto) + .ToFrozenDictionary(); + + var actual = fileResources.MapValue(normalizeDto) + .ToFrozenDictionary(); + + actual.Should().BeEquivalentTo(expected); + }; + + static string normalizeDto(ApiDiagnosticDto dto) => + new + { + LoggerId = string.Join('/', dto.Properties.LoggerId?.Split('/')?.TakeLast(2)?.ToArray() ?? []), + AlwaysLog = dto.Properties.AlwaysLog ?? string.Empty, + Sampling = new + { + Type = dto.Properties.Sampling?.SamplingType ?? string.Empty, + Percentage = dto.Properties.Sampling?.Percentage ?? 0 + } + }.ToString()!; + } + + public static void ConfigureGetApimApiDiagnostics(IHostApplicationBuilder builder) + { + ManagementServiceModule.ConfigureGetManagementServiceUri(builder); + AzureModule.ConfigureHttpPipeline(builder); + + builder.Services.TryAddSingleton(GetGetApimApiDiagnostics); + } + + private static GetApimApiDiagnostics GetGetApimApiDiagnostics(IServiceProvider provider) + { + var getServiceUri = provider.GetRequiredService(); + var pipeline = provider.GetRequiredService(); + var activitySource = provider.GetRequiredService(); + var logger = provider.GetRequiredService(); + + return async (apiName, serviceName, cancellationToken) => + { + using var _ = activitySource.StartActivity(nameof(GetApimApiDiagnostics)); + + logger.LogInformation("Getting diagnostics from API {ApiName} in {ServiceName}...", apiName, serviceName); + + var serviceUri = getServiceUri(serviceName); + var diagnosticsUri = ApiDiagnosticsUri.From(apiName, serviceUri); + + return await diagnosticsUri.List(pipeline, cancellationToken).ToFrozenDictionary(cancellationToken); + }; + } + + public static void ConfigureGetFileApiDiagnostics(IHostApplicationBuilder builder) + { + builder.Services.TryAddSingleton(GetGetFileApiDiagnostics); + } + + private static GetFileApiDiagnostics GetGetFileApiDiagnostics(IServiceProvider provider) + { + var activitySource = provider.GetRequiredService(); + var logger = provider.GetRequiredService(); + + return async (apiName, serviceDirectory, commitIdOption, cancellationToken) => + { + using var _ = activitySource.StartActivity(nameof(GetFileApiDiagnostics)); + + return await commitIdOption.Map(commitId => getWithCommit(apiName, serviceDirectory, commitId, cancellationToken)) + .IfNone(() => getWithoutCommit(apiName, serviceDirectory, cancellationToken)); + }; + + async ValueTask> getWithCommit(ApiName apiName, ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + logger.LogInformation("Getting diagnostics from API {ApiName} in {ServiceDirectory} as of commit {CommitId}...", apiName, serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => ApiDiagnosticInformationFile.TryParse(file, serviceDirectory)) + .Where(file => file.Parent.Parent.Parent.Name == apiName) + .Choose(async file => await tryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + static async ValueTask> tryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, ApiDiagnosticInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + async ValueTask> getWithoutCommit(ApiName apiName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + logger.LogInformation("Getting diagnostics from API {ApiName} in {ServiceDirectory}...", apiName, serviceDirectory); + + var informationFiles = common.ApiDiagnosticModule.ListInformationFiles(serviceDirectory); + + return await informationFiles.ToAsyncEnumerable() + .Where(file => file.Parent.Parent.Parent.Name == apiName) + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } + } + + public static void ConfigureWriteApiDiagnosticModels(IHostApplicationBuilder builder) + { + builder.Services.TryAddSingleton(GetWriteApiDiagnosticModels); + } + + private static WriteApiDiagnosticModels GetWriteApiDiagnosticModels(IServiceProvider provider) + { + var activitySource = provider.GetRequiredService(); + var logger = provider.GetRequiredService(); + + return async (models, apiName, serviceDirectory, cancellationToken) => + { + using var _ = activitySource.StartActivity(nameof(WriteApiDiagnosticModels)); + + logger.LogInformation("Writing diagnostic models for API {ApiName} in {ServiceDirectory}...", apiName, serviceDirectory); + + await models.IterParallel(async model => + { + await writeInformationFile(model, apiName, serviceDirectory, cancellationToken); + }, cancellationToken); + }; + + async ValueTask writeInformationFile(ApiDiagnosticModel model, ApiName apiName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + var informationFile = ApiDiagnosticInformationFile.From(model.Name, apiName, serviceDirectory); + var dto = getDto(model); + + await informationFile.WriteDto(dto, cancellationToken); + } + + static ApiDiagnosticDto getDto(ApiDiagnosticModel model) => + new() + { + Properties = new ApiDiagnosticDto.DiagnosticContract + { + LoggerId = $"/loggers/{model.LoggerName}", + AlwaysLog = model.AlwaysLog.ValueUnsafe(), + Sampling = model.Sampling.Map(sampling => new ApiDiagnosticDto.SamplingSettings + { + SamplingType = sampling.Type, + Percentage = sampling.Percentage + }).ValueUnsafe() + } + }; + } + public static void ConfigureValidatePublishedApiDiagnostics(IHostApplicationBuilder builder) + { + ConfigureGetApimApiDiagnostics(builder); + ConfigureGetFileApiDiagnostics(builder); + + builder.Services.TryAddSingleton(GetValidatePublishedApiDiagnostics); + } + + private static ValidatePublishedApiDiagnostics GetValidatePublishedApiDiagnostics(IServiceProvider provider) + { + var getApimResources = provider.GetRequiredService(); + var getFileResources = provider.GetRequiredService(); + var activitySource = provider.GetRequiredService(); + var logger = provider.GetRequiredService(); + + return async (apiName, commitIdOption, serviceName, serviceDirectory, cancellationToken) => + { + using var _ = activitySource.StartActivity(nameof(ValidatePublishedApiDiagnostics)); + + logger.LogInformation("Validating published diagnostics in API {ApiName} in {ServiceDirectory}...", apiName, serviceDirectory); + + var apimResources = await getApimResources(apiName, serviceName, cancellationToken); + var fileResources = await getFileResources(apiName, serviceDirectory, commitIdOption, cancellationToken); + + var expected = fileResources.MapValue(normalizeDto) + .ToFrozenDictionary(); + + var actual = apimResources.MapValue(normalizeDto) + .ToFrozenDictionary(); + + actual.Should().BeEquivalentTo(expected); + }; + + static string normalizeDto(ApiDiagnosticDto dto) => + new + { + LoggerId = string.Join('/', dto.Properties.LoggerId?.Split('/')?.TakeLast(2)?.ToArray() ?? []), + AlwaysLog = dto.Properties.AlwaysLog ?? string.Empty, + Sampling = new + { + Type = dto.Properties.Sampling?.SamplingType ?? string.Empty, + Percentage = dto.Properties.Sampling?.Percentage ?? 0 + } + }.ToString()!; + } +} +//internal static class ApiDiagnosticModule +//{ +// public static async ValueTask Put(IEnumerable models, ApiName apiName, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => +// await models.IterParallel(async model => +// { +// await Put(model, apiName, serviceUri, pipeline, cancellationToken); +// }, cancellationToken); + +// private static async ValueTask Put(ApiDiagnosticModel model, ApiName apiName, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +// { +// var uri = ApiDiagnosticUri.From(model.Name, apiName, serviceUri); +// var dto = ModelToDto(model); +// await uri.PutDto(dto, pipeline, cancellationToken); +// } + +// private static ApiDiagnosticDto ModelToDto(ApiDiagnosticModel model) => +// new() +// { +// Properties = new ApiDiagnosticDto.DiagnosticContract +// { +// LoggerId = $"/loggers/{model.LoggerName}", +// AlwaysLog = model.AlwaysLog.ValueUnsafe(), +// Sampling = model.Sampling.Map(sampling => new ApiDiagnosticDto.SamplingSettings +// { +// SamplingType = sampling.Type, +// Percentage = sampling.Percentage +// }).ValueUnsafe() +// } +// }; + +// public static async ValueTask WriteArtifacts(IEnumerable models, ApiName apiName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => +// await models.IterParallel(async model => +// { +// await WriteInformationFile(model, apiName, serviceDirectory, cancellationToken); +// }, cancellationToken); + +// private static async ValueTask WriteInformationFile(ApiDiagnosticModel model, ApiName apiName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) +// { +// var informationFile = ApiDiagnosticInformationFile.From(model.Name, apiName, serviceDirectory); +// var dto = ModelToDto(model); +// await informationFile.WriteDto(dto, cancellationToken); +// } + +// public static async ValueTask ValidateExtractedArtifacts(ManagementServiceDirectory serviceDirectory, ApiName apiName, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +// { +// var apimResources = await GetApimResources(apiName, serviceUri, pipeline, cancellationToken); +// var fileResources = await GetFileResources(apiName, serviceDirectory, cancellationToken); + +// var expected = apimResources.MapValue(NormalizeDto); +// var actual = fileResources.MapValue(NormalizeDto); + +// actual.Should().BeEquivalentTo(expected); +// } + +// private static async ValueTask> GetApimResources(ApiName apiName, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +// { +// var uri = ApiDiagnosticsUri.From(apiName, serviceUri); + +// return await uri.List(pipeline, cancellationToken) +// .ToFrozenDictionary(cancellationToken); +// } + +// private static async ValueTask> GetFileResources(ApiName apiName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => +// await common.ApiDiagnosticModule.ListInformationFiles(serviceDirectory) +// .Where(file => file.Parent.Parent.Parent.Name == apiName) +// .ToAsyncEnumerable() +// .SelectAwait(async file => (file.Parent.Name, +// await file.ReadDto(cancellationToken))) +// .ToFrozenDictionary(cancellationToken); + +// private static string NormalizeDto(ApiDiagnosticDto dto) => +// new +// { +// LoggerId = string.Join('/', dto.Properties.LoggerId?.Split('/')?.TakeLast(2)?.ToArray() ?? []), +// AlwaysLog = dto.Properties.AlwaysLog ?? string.Empty, +// Sampling = new +// { +// Type = dto.Properties.Sampling?.SamplingType ?? string.Empty, +// Percentage = dto.Properties.Sampling?.Percentage ?? 0 +// } +// }.ToString()!; + +// public static async ValueTask ValidatePublisherChanges(ApiName apiName, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +// { +// var fileResources = await GetFileResources(apiName, serviceDirectory, cancellationToken); +// await ValidatePublisherChanges(apiName, fileResources, serviceUri, pipeline, cancellationToken); +// } + +// private static async ValueTask ValidatePublisherChanges(ApiName apiName, IDictionary fileResources, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +// { +// var apimResources = await GetApimResources(apiName, serviceUri, pipeline, cancellationToken); + +// var expected = fileResources.MapValue(NormalizeDto); +// var actual = apimResources.MapValue(NormalizeDto); +// actual.Should().BeEquivalentTo(expected); +// } + +// public static async ValueTask ValidatePublisherCommitChanges(ApiName apiName, CommitId commitId, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +// { +// var fileResources = await GetFileResources(apiName, commitId, serviceDirectory, cancellationToken); +// await ValidatePublisherChanges(apiName, fileResources, serviceUri, pipeline, cancellationToken); +// } + +// private static async ValueTask> GetFileResources(ApiName apiName, CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => +// await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) +// .ToAsyncEnumerable() +// .Choose(file => ApiDiagnosticInformationFile.TryParse(file, serviceDirectory)) +// .Where(file => file.Parent.Parent.Parent.Name == apiName) +// .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) +// .ToFrozenDictionary(cancellationToken); + +// private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, ApiDiagnosticInformationFile file, CancellationToken cancellationToken) +// { +// var name = file.Parent.Name; +// var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + +// return await contentsOption.MapTask(async contents => +// { +// using (contents) +// { +// var data = await BinaryData.FromStreamAsync(contents, cancellationToken); +// var dto = data.ToObjectFromJson(); +// return (name, dto); +// } +// }); +// } +//} diff --git a/tools/code/integration.tests/Test.cs b/tools/code/integration.tests/Test.cs index 7e93c131..67eeb14c 100644 --- a/tools/code/integration.tests/Test.cs +++ b/tools/code/integration.tests/Test.cs @@ -267,7 +267,7 @@ from policyFragments in Generator.GenerateNewSet(initialModel.PolicyFragments, P from servicePolicies in Generator.GenerateNewSet(initialModel.ServicePolicies, ServicePolicyModel.GenerateSet(), ServicePolicyModule.GenerateUpdate) from groups in Generator.GenerateNewSet(initialModel.Groups, GroupModel.GenerateSet(), GroupModule.GenerateUpdate) from apis in from apiSet in Generator.GenerateNewSet(initialModel.Apis, ApiModel.GenerateSet(), ApiModule.GenerateUpdate) - from updatedApis in ServiceModel.UpdateApis(apiSet, versionSets, tags) + from updatedApis in ServiceModel.UpdateApis(apiSet, versionSets, tags, loggers) select updatedApis from products in from productSet in Generator.GenerateNewSet(initialModel.Products, ProductModel.GenerateSet(), ProductModule.GenerateUpdate) from updatedProductGroups in ServiceModel.UpdateProducts(productSet, groups, tags, apis) diff --git a/tools/code/publisher/ApiDiagnostic.cs b/tools/code/publisher/ApiDiagnostic.cs new file mode 100644 index 00000000..5a726612 --- /dev/null +++ b/tools/code/publisher/ApiDiagnostic.cs @@ -0,0 +1,249 @@ +using Azure.Core.Pipeline; +using common; +using LanguageExt; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using System; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace publisher; + +public delegate ValueTask PutApiDiagnostics(CancellationToken cancellationToken); +public delegate ValueTask DeleteApiDiagnostics(CancellationToken cancellationToken); +public delegate Option<(ApiDiagnosticName Name, ApiName ApiName)> TryParseApiDiagnosticName(FileInfo file); +public delegate bool IsApiDiagnosticNameInSourceControl(ApiDiagnosticName name, ApiName apiName); +public delegate ValueTask PutApiDiagnostic(ApiDiagnosticName name, ApiName apiName, CancellationToken cancellationToken); +public delegate ValueTask> FindApiDiagnosticDto(ApiDiagnosticName name, ApiName apiName, CancellationToken cancellationToken); +public delegate ValueTask PutApiDiagnosticInApim(ApiDiagnosticName name, ApiDiagnosticDto dto, ApiName apiName, CancellationToken cancellationToken); +public delegate ValueTask DeleteApiDiagnostic(ApiDiagnosticName name, ApiName apiName, CancellationToken cancellationToken); +public delegate ValueTask DeleteApiDiagnosticFromApim(ApiDiagnosticName name, ApiName apiName, CancellationToken cancellationToken); + +internal static class ApiDiagnosticModule +{ + public static void ConfigurePutApiDiagnostics(IHostApplicationBuilder builder) + { + CommonModule.ConfigureGetPublisherFiles(builder); + ConfigureTryParseApiDiagnosticName(builder); + ConfigureIsApiDiagnosticNameInSourceControl(builder); + ConfigurePutApiDiagnostic(builder); + + builder.Services.TryAddSingleton(GetPutApiDiagnostics); + } + + private static PutApiDiagnostics GetPutApiDiagnostics(IServiceProvider provider) + { + var getPublisherFiles = provider.GetRequiredService(); + var tryParseName = provider.GetRequiredService(); + var isNameInSourceControl = provider.GetRequiredService(); + var put = provider.GetRequiredService(); + var activitySource = provider.GetRequiredService(); + var logger = provider.GetRequiredService(); + + return async cancellationToken => + { + using var _ = activitySource.StartActivity(nameof(PutApiDiagnostics)); + + logger.LogInformation("Putting diagnostics..."); + + await getPublisherFiles() + .Choose(tryParseName.Invoke) + .Where(resource => isNameInSourceControl(resource.Name, resource.ApiName)) + .Distinct() + .IterParallel(put.Invoke, cancellationToken); + }; + } + + private static void ConfigureTryParseApiDiagnosticName(IHostApplicationBuilder builder) + { + AzureModule.ConfigureManagementServiceDirectory(builder); + + builder.Services.TryAddSingleton(GetTryParseApiDiagnosticName); + } + + private static TryParseApiDiagnosticName GetTryParseApiDiagnosticName(IServiceProvider provider) + { + var serviceDirectory = provider.GetRequiredService(); + + return file => from informationFile in ApiDiagnosticInformationFile.TryParse(file, serviceDirectory) + select (informationFile.Parent.Name, informationFile.Parent.Parent.Parent.Name); + } + + private static void ConfigureIsApiDiagnosticNameInSourceControl(IHostApplicationBuilder builder) + { + CommonModule.ConfigureGetArtifactFiles(builder); + AzureModule.ConfigureManagementServiceDirectory(builder); + + builder.Services.TryAddSingleton(GetIsApiDiagnosticNameInSourceControl); + } + + private static IsApiDiagnosticNameInSourceControl GetIsApiDiagnosticNameInSourceControl(IServiceProvider provider) + { + var getArtifactFiles = provider.GetRequiredService(); + var serviceDirectory = provider.GetRequiredService(); + + return doesInformationFileExist; + + bool doesInformationFileExist(ApiDiagnosticName name, ApiName apiName) + { + var artifactFiles = getArtifactFiles(); + var informationFile = ApiDiagnosticInformationFile.From(name, apiName, serviceDirectory); + + return artifactFiles.Contains(informationFile.ToFileInfo()); + } + } + + private static void ConfigurePutApiDiagnostic(IHostApplicationBuilder builder) + { + ConfigureFindApiDiagnosticDto(builder); + ConfigurePutApiDiagnosticInApim(builder); + + builder.Services.TryAddSingleton(GetPutApiDiagnostic); + } + + private static PutApiDiagnostic GetPutApiDiagnostic(IServiceProvider provider) + { + var findDto = provider.GetRequiredService(); + var putInApim = provider.GetRequiredService(); + var activitySource = provider.GetRequiredService(); + + return async (name, apiName, cancellationToken) => + { + using var _ = activitySource.StartActivity(nameof(PutApiDiagnostic)) + ?.AddTag("api.name", apiName) + ?.AddTag("api_diagnostic.name", name); + + var dtoOption = await findDto(name, apiName, cancellationToken); + await dtoOption.IterTask(async dto => await putInApim(name, dto, apiName, cancellationToken)); + }; + } + + private static void ConfigureFindApiDiagnosticDto(IHostApplicationBuilder builder) + { + AzureModule.ConfigureManagementServiceDirectory(builder); + CommonModule.ConfigureTryGetFileContents(builder); + + builder.Services.TryAddSingleton(GetFindApiDiagnosticDto); + } + + private static FindApiDiagnosticDto GetFindApiDiagnosticDto(IServiceProvider provider) + { + var serviceDirectory = provider.GetRequiredService(); + var tryGetFileContents = provider.GetRequiredService(); + + return async (name, apiName, cancellationToken) => + { + var informationFile = ApiDiagnosticInformationFile.From(name, apiName, serviceDirectory); + var contentsOption = await tryGetFileContents(informationFile.ToFileInfo(), cancellationToken); + + return from contents in contentsOption + select contents.ToObjectFromJson(); + }; + } + + private static void ConfigurePutApiDiagnosticInApim(IHostApplicationBuilder builder) + { + AzureModule.ConfigureManagementServiceUri(builder); + AzureModule.ConfigureHttpPipeline(builder); + + builder.Services.TryAddSingleton(GetPutApiDiagnosticInApim); + } + + private static PutApiDiagnosticInApim GetPutApiDiagnosticInApim(IServiceProvider provider) + { + var serviceUri = provider.GetRequiredService(); + var pipeline = provider.GetRequiredService(); + var logger = provider.GetRequiredService(); + + return async (name, dto, apiName, cancellationToken) => + { + logger.LogInformation("Adding diagnostic {ApiDiagnosticName} to API {ApiName}...", name, apiName); + + var resourceUri = ApiDiagnosticUri.From(name, apiName, serviceUri); + await resourceUri.PutDto(dto, pipeline, cancellationToken); + }; + } + + public static void ConfigureDeleteApiDiagnostics(IHostApplicationBuilder builder) + { + CommonModule.ConfigureGetPublisherFiles(builder); + ConfigureTryParseApiDiagnosticName(builder); + ConfigureIsApiDiagnosticNameInSourceControl(builder); + ConfigureDeleteApiDiagnostic(builder); + + builder.Services.TryAddSingleton(GetDeleteApiDiagnostics); + } + + private static DeleteApiDiagnostics GetDeleteApiDiagnostics(IServiceProvider provider) + { + var getPublisherFiles = provider.GetRequiredService(); + var tryParseName = provider.GetRequiredService(); + var isNameInSourceControl = provider.GetRequiredService(); + var delete = provider.GetRequiredService(); + var activitySource = provider.GetRequiredService(); + var logger = provider.GetRequiredService(); + + return async cancellationToken => + { + using var _ = activitySource.StartActivity(nameof(DeleteApiDiagnostics)); + + logger.LogInformation("Deleting diagnostics..."); + + await getPublisherFiles() + .Choose(tryParseName.Invoke) + .Where(resource => isNameInSourceControl(resource.Name, resource.ApiName) is false) + .Distinct() + .IterParallel(delete.Invoke, cancellationToken); + }; + } + + private static void ConfigureDeleteApiDiagnostic(IHostApplicationBuilder builder) + { + ConfigureDeleteApiDiagnosticFromApim(builder); + + builder.Services.TryAddSingleton(GetDeleteApiDiagnostic); + } + + private static DeleteApiDiagnostic GetDeleteApiDiagnostic(IServiceProvider provider) + { + var deleteFromApim = provider.GetRequiredService(); + var activitySource = provider.GetRequiredService(); + + return async (name, apiName, cancellationToken) => + { + using var _ = activitySource.StartActivity(nameof(DeleteApiDiagnostic)) + ?.AddTag("api.name", apiName) + ?.AddTag("api_diagnostic.name", name); + + await deleteFromApim(name, apiName, cancellationToken); + }; + } + + private static void ConfigureDeleteApiDiagnosticFromApim(IHostApplicationBuilder builder) + { + AzureModule.ConfigureManagementServiceUri(builder); + AzureModule.ConfigureHttpPipeline(builder); + + builder.Services.TryAddSingleton(GetDeleteApiDiagnosticFromApim); + } + + private static DeleteApiDiagnosticFromApim GetDeleteApiDiagnosticFromApim(IServiceProvider provider) + { + var serviceUri = provider.GetRequiredService(); + var pipeline = provider.GetRequiredService(); + var logger = provider.GetRequiredService(); + + return async (name, apiName, cancellationToken) => + { + logger.LogInformation("Removing diagnostic {ApiDiagnosticName} from API {ApiName}...", name, apiName); + + var resourceUri = ApiDiagnosticUri.From(name, apiName, serviceUri); + await resourceUri.Delete(pipeline, cancellationToken); + }; + } +} \ No newline at end of file diff --git a/tools/code/publisher/App.cs b/tools/code/publisher/App.cs index 3d15d876..f01cdd29 100644 --- a/tools/code/publisher/App.cs +++ b/tools/code/publisher/App.cs @@ -28,6 +28,7 @@ public static void ConfigureRunApplication(IHostApplicationBuilder builder) SubscriptionModule.ConfigurePutSubscriptions(builder); ApiPolicyModule.ConfigurePutApiPolicies(builder); ApiTagModule.ConfigurePutApiTags(builder); + ApiDiagnosticModule.ConfigurePutApiDiagnostics(builder); GatewayApiModule.ConfigurePutGatewayApis(builder); ProductPolicyModule.ConfigurePutProductPolicies(builder); ProductGroupModule.ConfigurePutProductGroups(builder); @@ -62,6 +63,7 @@ public static void ConfigureRunApplication(IHostApplicationBuilder builder) ProductGroupModule.ConfigureDeleteProductGroups(builder); ProductPolicyModule.ConfigureDeleteProductPolicies(builder); GatewayApiModule.ConfigureDeleteGatewayApis(builder); + ApiDiagnosticModule.ConfigureDeleteApiDiagnostics(builder); ApiTagModule.ConfigureDeleteApiTags(builder); ApiPolicyModule.ConfigureDeleteApiPolicies(builder); SubscriptionModule.ConfigureDeleteSubscriptions(builder); @@ -99,6 +101,7 @@ private static RunApplication GetRunApplication(IServiceProvider provider) var putSubscriptions = provider.GetRequiredService(); var putApiPolicies = provider.GetRequiredService(); var putApiTags = provider.GetRequiredService(); + var putApiDiagnostics = provider.GetRequiredService(); var putGatewayApis = provider.GetRequiredService(); var putProductPolicies = provider.GetRequiredService(); var putProductGroups = provider.GetRequiredService(); @@ -133,6 +136,7 @@ private static RunApplication GetRunApplication(IServiceProvider provider) var deleteProductGroups = provider.GetRequiredService(); var deleteProductPolicies = provider.GetRequiredService(); var deleteGatewayApis = provider.GetRequiredService(); + var deleteApiDiagnostics = provider.GetRequiredService(); var deleteApiTags = provider.GetRequiredService(); var deleteApiPolicies = provider.GetRequiredService(); var deleteSubscriptions = provider.GetRequiredService(); @@ -173,6 +177,7 @@ private static RunApplication GetRunApplication(IServiceProvider provider) await putSubscriptions(cancellationToken); await putApiPolicies(cancellationToken); await putApiTags(cancellationToken); + await putApiDiagnostics(cancellationToken); await putGatewayApis(cancellationToken); await putProductPolicies(cancellationToken); await putProductGroups(cancellationToken); @@ -212,6 +217,7 @@ private static RunApplication GetRunApplication(IServiceProvider provider) await deleteProductGroups(cancellationToken); await deleteProductPolicies(cancellationToken); await deleteGatewayApis(cancellationToken); + await deleteApiDiagnostics(cancellationToken); await deleteApiTags(cancellationToken); await deleteApiPolicies(cancellationToken); await deleteSubscriptions(cancellationToken); diff --git a/tools/code/publisher/publisher.csproj b/tools/code/publisher/publisher.csproj index 72c4254a..9d8f66ba 100644 --- a/tools/code/publisher/publisher.csproj +++ b/tools/code/publisher/publisher.csproj @@ -1,24 +1,24 @@  - - net8.0 - true - 8-all - CA1708,CA1724,CA1812,CA1848,CA2007,CA1034,CA1062 - Exe - enable - 6ce9dc18-ea0d-4d82-8f1a-e63877f35b88 - 6.0.1 - + + net8.0 + true + 8-all + CA1708,CA1724,CA1812,CA1848,CA2007,CA1034,CA1062 + Exe + enable + 6ce9dc18-ea0d-4d82-8f1a-e63877f35b88 + 6.0.1.1 + - - - - - + + + + + - - - + + +