From eea6a82075e0501919a1b3cc33377cb34f11a4a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Sharma?= Date: Tue, 3 Sep 2019 17:20:49 -0700 Subject: [PATCH] [Client SDK] Add catalog client (#321) Add the low level catalog resource client. Part of https://github.com/loic-sharma/BaGet/issues/308 --- .../DatabasePackageMetadataService.cs | 39 ++- src/BaGet.Protocol/BaGet.Protocol.csproj | 4 +- src/BaGet.Protocol/Catalog/CatalogClient.cs | 67 ++++++ src/BaGet.Protocol/Catalog/CatalogIndex.cs | 24 ++ src/BaGet.Protocol/Catalog/CatalogLeaf.cs | 37 +++ src/BaGet.Protocol/Catalog/CatalogLeafItem.cs | 30 +++ src/BaGet.Protocol/Catalog/CatalogLeafType.cs | 19 ++ .../Catalog/CatalogModelExtensions.cs | 222 ++++++++++++++++++ src/BaGet.Protocol/Catalog/CatalogPage.cs | 27 +++ src/BaGet.Protocol/Catalog/CatalogPageItem.cs | 22 ++ .../Catalog/ICatalogLeafItem.cs | 18 ++ .../Catalog/ICatalogResource.cs | 54 +++++ .../Catalog/PackageDeleteCatalogLeaf.cs | 12 + .../Catalog/PackageDetailsCatalogLeaf.cs | 94 ++++++++ .../Converters/BaseCatalogLeafConverter.cs | 35 +++ .../CatalogLeafItemTypeConverter.cs | 41 ++++ .../Converters/CatalogLeafTypeConverter.cs | 50 ++++ .../PackageDependencyRangeConverter.cs | 33 +++ src/BaGet.Protocol/INuGetClientFactory.cs | 7 + src/BaGet.Protocol/NuGetClientFactory.cs | 17 +- .../PackageMetadata/DependencyGroupItem.cs | 18 ++ .../PackageMetadata/DependencyItem.cs | 19 ++ .../PackageMetadata/PackageMetadata.cs | 49 ---- .../CatalogClientTests.cs | 55 +++++ .../Support/ProtocolFixture.cs | 2 + tests/BaGet.Tests/BaGet.Tests.csproj | 2 +- 26 files changed, 924 insertions(+), 73 deletions(-) create mode 100644 src/BaGet.Protocol/Catalog/CatalogClient.cs create mode 100644 src/BaGet.Protocol/Catalog/CatalogIndex.cs create mode 100644 src/BaGet.Protocol/Catalog/CatalogLeaf.cs create mode 100644 src/BaGet.Protocol/Catalog/CatalogLeafItem.cs create mode 100644 src/BaGet.Protocol/Catalog/CatalogLeafType.cs create mode 100644 src/BaGet.Protocol/Catalog/CatalogModelExtensions.cs create mode 100644 src/BaGet.Protocol/Catalog/CatalogPage.cs create mode 100644 src/BaGet.Protocol/Catalog/CatalogPageItem.cs create mode 100644 src/BaGet.Protocol/Catalog/ICatalogLeafItem.cs create mode 100644 src/BaGet.Protocol/Catalog/ICatalogResource.cs create mode 100644 src/BaGet.Protocol/Catalog/PackageDeleteCatalogLeaf.cs create mode 100644 src/BaGet.Protocol/Catalog/PackageDetailsCatalogLeaf.cs create mode 100644 src/BaGet.Protocol/Converters/BaseCatalogLeafConverter.cs create mode 100644 src/BaGet.Protocol/Converters/CatalogLeafItemTypeConverter.cs create mode 100644 src/BaGet.Protocol/Converters/CatalogLeafTypeConverter.cs create mode 100644 src/BaGet.Protocol/Converters/PackageDependencyRangeConverter.cs create mode 100644 src/BaGet.Protocol/PackageMetadata/DependencyGroupItem.cs create mode 100644 src/BaGet.Protocol/PackageMetadata/DependencyItem.cs create mode 100644 tests/BaGet.Protocol.Tests/CatalogClientTests.cs diff --git a/src/BaGet.Core/Metadata/DatabasePackageMetadataService.cs b/src/BaGet.Core/Metadata/DatabasePackageMetadataService.cs index 5d5f08d7..f61540a4 100644 --- a/src/BaGet.Core/Metadata/DatabasePackageMetadataService.cs +++ b/src/BaGet.Core/Metadata/DatabasePackageMetadataService.cs @@ -5,7 +5,6 @@ using System.Threading.Tasks; using BaGet.Core.Entities; using BaGet.Core.Mirror; -using BaGet.Core.ServiceIndex; using BaGet.Protocol; using NuGet.Versioning; @@ -130,25 +129,25 @@ private RegistrationIndexPageItem ToRegistrationIndexPageItem(Package package) = private IReadOnlyList ToDependencyGroups(Package package) { - var groups = new List(); - - var targetFrameworks = package.Dependencies.Select(d => d.TargetFramework).Distinct(); - - foreach (var target in targetFrameworks) - { - // A package may have no dependencies for a target framework. This is represented - // by a single dependency item with a null "Id" and "VersionRange". - var groupId = $"https://api.nuget.org/v3/catalog0/data/2015.02.01.06.24.15/{package.Id}.{package.Version}.json#dependencygroup/{target}"; - var dependencyItems = package.Dependencies - .Where(d => d.TargetFramework == target) - .Where(d => d.Id != null && d.VersionRange != null) - .Select(d => new DependencyItem($"{groupId}/{d.Id}", d.Id, d.VersionRange)) - .ToList(); - - groups.Add(new DependencyGroupItem(groupId, target, dependencyItems)); - } - - return groups; + return package.Dependencies + .GroupBy(d => d.TargetFramework) + .Select(group => new DependencyGroupItem + { + TargetFramework = group.Key, + + // A package that supports a target framework but does not have dependencies while on + // that target framework is represented by a fake dependency with a null "Id" and "VersionRange". + // This fake dependency should not be included in the output. + Dependencies = group + .Where(d => d.Id != null && d.VersionRange != null) + .Select(d => new DependencyItem + { + Id = d.Id, + Range = d.VersionRange + }) + .ToList() + }) + .ToList(); } } } diff --git a/src/BaGet.Protocol/BaGet.Protocol.csproj b/src/BaGet.Protocol/BaGet.Protocol.csproj index 0a0ea8be..ac3dc3af 100644 --- a/src/BaGet.Protocol/BaGet.Protocol.csproj +++ b/src/BaGet.Protocol/BaGet.Protocol.csproj @@ -1,4 +1,4 @@ - + netstandard2.0 @@ -9,7 +9,7 @@ - + diff --git a/src/BaGet.Protocol/Catalog/CatalogClient.cs b/src/BaGet.Protocol/Catalog/CatalogClient.cs new file mode 100644 index 00000000..7a531b3f --- /dev/null +++ b/src/BaGet.Protocol/Catalog/CatalogClient.cs @@ -0,0 +1,67 @@ +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace BaGet.Protocol.Internal +{ + public class CatalogClient : ICatalogResource + { + private readonly HttpClient _httpClient; + private readonly string _catalogUrl; + + public CatalogClient(HttpClient httpClient, string catalogUrl) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + _catalogUrl = catalogUrl ?? throw new ArgumentNullException(nameof(catalogUrl)); + } + + public async Task GetIndexAsync(CancellationToken cancellationToken = default) + { + var response = await _httpClient.DeserializeUrlAsync(_catalogUrl, cancellationToken); + + return response.GetResultOrThrow(); + } + + public async Task GetPageAsync(string pageUrl, CancellationToken cancellationToken = default) + { + var response = await _httpClient.DeserializeUrlAsync(pageUrl, cancellationToken); + + return response.GetResultOrThrow(); + } + + public async Task GetPackageDeleteLeafAsync(string leafUrl, CancellationToken cancellationToken = default) + { + return await GetAndValidateLeafAsync( + CatalogLeafType.PackageDelete, + leafUrl, + cancellationToken); + } + + public async Task GetPackageDetailsLeafAsync(string leafUrl, CancellationToken cancellationToken = default) + { + return await GetAndValidateLeafAsync( + CatalogLeafType.PackageDetails, + leafUrl, + cancellationToken); + } + + private async Task GetAndValidateLeafAsync( + CatalogLeafType type, + string leafUrl, + CancellationToken cancellationToken) where T : CatalogLeaf + { + var result = await _httpClient.DeserializeUrlAsync(leafUrl, cancellationToken); + var leaf = result.GetResultOrThrow(); + + if (leaf.Type != type) + { + throw new ArgumentException( + $"The leaf type found in the document does not match the expected '{type}' type.", + nameof(type)); + } + + return leaf; + } + } +} diff --git a/src/BaGet.Protocol/Catalog/CatalogIndex.cs b/src/BaGet.Protocol/Catalog/CatalogIndex.cs new file mode 100644 index 00000000..bd63b639 --- /dev/null +++ b/src/BaGet.Protocol/Catalog/CatalogIndex.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace BaGet.Protocol +{ + /// + /// The catalog index is the entry point for the catalog resource. + /// Use this to discover catalog pages, which in turn can be used to discover catalog leafs. + /// See: https://docs.microsoft.com/en-us/nuget/api/catalog-resource#catalog-index + /// Based off: https://github.com/NuGet/NuGet.Services.Metadata/blob/64af0b59c5a79e0143f0808b39946df9f16cb2e7/src/NuGet.Protocol.Catalog/Models/CatalogIndex.cs + /// + public class CatalogIndex + { + [JsonProperty("commitTimeStamp")] + public DateTimeOffset CommitTimestamp { get; set; } + + [JsonProperty("count")] + public int Count { get; set; } + + [JsonProperty("items")] + public List Items { get; set; } + } +} diff --git a/src/BaGet.Protocol/Catalog/CatalogLeaf.cs b/src/BaGet.Protocol/Catalog/CatalogLeaf.cs new file mode 100644 index 00000000..4ab8c3a3 --- /dev/null +++ b/src/BaGet.Protocol/Catalog/CatalogLeaf.cs @@ -0,0 +1,37 @@ +using System; +using BaGet.Protocol.Internal; +using Newtonsoft.Json; + +namespace BaGet.Protocol +{ + /// + /// A catalog leaf. Represents a single package event. + /// Leafs can be discovered from a . + /// See: https://docs.microsoft.com/en-us/nuget/api/catalog-resource#catalog-leaf + /// Based off: https://github.com/NuGet/NuGet.Services.Metadata/blob/64af0b59c5a79e0143f0808b39946df9f16cb2e7/src/NuGet.Protocol.Catalog/Models/CatalogLeaf.cs + /// + public class CatalogLeaf : ICatalogLeafItem + { + [JsonProperty("@id")] + public string Url { get; set; } + + [JsonProperty("@type")] + [JsonConverter(typeof(CatalogLeafTypeConverter))] + public CatalogLeafType Type { get; set; } + + [JsonProperty("catalog:commitId")] + public string CommitId { get; set; } + + [JsonProperty("catalog:commitTimeStamp")] + public DateTimeOffset CommitTimestamp { get; set; } + + [JsonProperty("id")] + public string PackageId { get; set; } + + [JsonProperty("published")] + public DateTimeOffset Published { get; set; } + + [JsonProperty("version")] + public string PackageVersion { get; set; } + } +} diff --git a/src/BaGet.Protocol/Catalog/CatalogLeafItem.cs b/src/BaGet.Protocol/Catalog/CatalogLeafItem.cs new file mode 100644 index 00000000..4ae24b6f --- /dev/null +++ b/src/BaGet.Protocol/Catalog/CatalogLeafItem.cs @@ -0,0 +1,30 @@ +using System; +using BaGet.Protocol.Internal; +using Newtonsoft.Json; + +namespace BaGet.Protocol +{ + /// + /// An item in a that references a . + /// See: https://docs.microsoft.com/en-us/nuget/api/catalog-resource#catalog-item-object-in-a-page + /// Based off: https://github.com/NuGet/NuGet.Services.Metadata/blob/64af0b59c5a79e0143f0808b39946df9f16cb2e7/src/NuGet.Protocol.Catalog/Models/CatalogLeafItem.cs + /// + public class CatalogLeafItem : ICatalogLeafItem + { + [JsonProperty("@id")] + public string Url { get; set; } + + [JsonProperty("@type")] + [JsonConverter(typeof(CatalogLeafItemTypeConverter))] + public CatalogLeafType Type { get; set; } + + [JsonProperty("commitTimeStamp")] + public DateTimeOffset CommitTimestamp { get; set; } + + [JsonProperty("nuget:id")] + public string PackageId { get; set; } + + [JsonProperty("nuget:version")] + public string PackageVersion { get; set; } + } +} diff --git a/src/BaGet.Protocol/Catalog/CatalogLeafType.cs b/src/BaGet.Protocol/Catalog/CatalogLeafType.cs new file mode 100644 index 00000000..c19d7a65 --- /dev/null +++ b/src/BaGet.Protocol/Catalog/CatalogLeafType.cs @@ -0,0 +1,19 @@ +namespace BaGet.Protocol +{ + /// + /// The type of a . + /// Based off: https://github.com/NuGet/NuGet.Services.Metadata/blob/64af0b59c5a79e0143f0808b39946df9f16cb2e7/src/NuGet.Protocol.Catalog/Models/CatalogLeafType.cs + /// + public enum CatalogLeafType + { + /// + /// The represents the snapshot of a package's metadata. + /// + PackageDetails = 1, + + /// + /// The represents a package that was deleted. + /// + PackageDelete = 2, + } +} diff --git a/src/BaGet.Protocol/Catalog/CatalogModelExtensions.cs b/src/BaGet.Protocol/Catalog/CatalogModelExtensions.cs new file mode 100644 index 00000000..0132a4b3 --- /dev/null +++ b/src/BaGet.Protocol/Catalog/CatalogModelExtensions.cs @@ -0,0 +1,222 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using NuGet.Frameworks; +using NuGet.Packaging.Core; +using NuGet.Versioning; + +namespace BaGet.Protocol +{ + /// + /// These are documented interpretations of values returned by the catalog API. + /// Based off: https://github.com/NuGet/NuGet.Services.Metadata/blob/64af0b59c5a79e0143f0808b39946df9f16cb2e7/src/NuGet.Protocol.Catalog/Models/ModelExtensions.cs + /// + public static class CatalogModelExtensions + { + /// + /// Gets the leaves that lie within the provided commit timestamp bounds. The result is sorted by commit + /// timestamp, then package ID, then package version (SemVer order). + /// + /// + /// The exclusive lower time bound on . + /// The inclusive upper time bound on . + /// Only show the latest leaf concerning each package. + public static List GetLeavesInBounds( + this CatalogPage catalogPage, + DateTimeOffset minCommitTimestamp, + DateTimeOffset maxCommitTimestamp, + bool excludeRedundantLeaves) + { + var leaves = catalogPage + .Items + .Where(x => x.CommitTimestamp > minCommitTimestamp && x.CommitTimestamp <= maxCommitTimestamp) + .OrderBy(x => x.CommitTimestamp); + + if (excludeRedundantLeaves) + { + leaves = leaves + .GroupBy(x => new PackageIdentity(x.PackageId, x.ParsePackageVersion())) + .Select(x => x.Last()) + .OrderBy(x => x.CommitTimestamp); + } + + return leaves + .ThenBy(x => x.PackageId, StringComparer.OrdinalIgnoreCase) + .ThenBy(x => x.ParsePackageVersion()) + .ToList(); + } + + /// + /// Gets the pages that may have catalog leaves within the provided commit timestamp bounds. The result is + /// sorted by commit timestamp. + /// + /// The catalog index to fetch pages from. + /// The exclusive lower time bound on . + /// The inclusive upper time bound on . + public static List GetPagesInBounds( + this CatalogIndex catalogIndex, + DateTimeOffset minCommitTimestamp, + DateTimeOffset maxCommitTimestamp) + { + return catalogIndex + .GetPagesInBoundsLazy(minCommitTimestamp, maxCommitTimestamp) + .ToList(); + } + + private static IEnumerable GetPagesInBoundsLazy( + this CatalogIndex catalogIndex, + DateTimeOffset minCommitTimestamp, + DateTimeOffset maxCommitTimestamp) + { + // Filter out pages that fall entirely before the minimum commit timestamp and sort the remaining pages by + // commit timestamp. + var upperRange = catalogIndex + .Items + .Where(x => x.CommitTimestamp > minCommitTimestamp) + .OrderBy(x => x.CommitTimestamp); + + // Take pages from the sorted list until the commit timestamp goes past the maximum commit timestamp. This + // essentially LINQ's TakeWhile plus one more element. + foreach (var page in upperRange) + { + yield return page; + + if (page.CommitTimestamp > maxCommitTimestamp) + { + break; + } + } + } + + /// + /// Parse the package version as a . + /// + /// The catalog leaf. + /// The package version. + public static NuGetVersion ParsePackageVersion(this ICatalogLeafItem leaf) + { + return NuGetVersion.Parse(leaf.PackageVersion); + } + + /// + /// Parse the target framework as a . + /// + /// The package dependency group. + /// The framework. + public static NuGetFramework ParseTargetFramework(this DependencyGroupItem packageDependencyGroup) + { + if (string.IsNullOrEmpty(packageDependencyGroup.TargetFramework)) + { + return NuGetFramework.AnyFramework; + } + + return NuGetFramework.Parse(packageDependencyGroup.TargetFramework); + } + + /// + /// Parse the version range as a . + /// + /// The package dependency. + /// The version range. + public static VersionRange ParseRange(this DependencyItem packageDependency) + { + // Server side treats invalid version ranges as empty strings. + // Source: https://github.com/NuGet/NuGet.Services.Metadata/blob/382c214c60993edfd7158bc6d223fafeebbc920c/src/Catalog/Helpers/NuGetVersionUtility.cs#L25-L34 + // Client side treats empty string version ranges as the "all" range. + // Source: https://github.com/NuGet/NuGet.Client/blob/849063018d8ee08625774a2dcd07ab84224dabb9/src/NuGet.Core/NuGet.Protocol/DependencyInfo/RegistrationUtility.cs#L20-L30 + // Example: https://api.nuget.org/v3/catalog0/data/2016.03.14.21.19.28/servicestack.extras.serilog.2.0.1.json + if (!VersionRange.TryParse(packageDependency.Range, out var parsed)) + { + return VersionRange.All; + } + + return parsed; + } + + /// + /// Determines if the provided catalog leaf is a package delete. + /// + /// The catalog leaf. + /// True if the catalog leaf represents a package delete. + public static bool IsPackageDelete(this ICatalogLeafItem leaf) + { + return leaf.Type == CatalogLeafType.PackageDelete; + } + + /// + /// Determines if the provided catalog leaf is contains package details. + /// + /// The catalog leaf. + /// True if the catalog leaf contains package details. + public static bool IsPackageDetails(this ICatalogLeafItem leaf) + { + return leaf.Type == CatalogLeafType.PackageDetails; + } + + /// + /// Determines if the provided package details leaf represents a listed package. + /// + /// The catalog leaf. + /// True if the package is listed. + public static bool IsListed(this PackageDetailsCatalogLeaf leaf) + { + if (leaf.Listed.HasValue) + { + return leaf.Listed.Value; + } + + // A published year of 1900 indicates that this package is unlisted, when the listed property itself is + // not present (legacy behavior). + // Example: https://api.nuget.org/v3/catalog0/data/2015.02.01.06.22.45/antixss.4.0.1.json + return leaf.Published.Year != 1900; + } + + /// + /// Determines if the provied package details leaf represents a SemVer 2.0.0 package. A package is considered + /// SemVer 2.0.0 if it's version is SemVer 2.0.0 or one of its dependency version ranges is SemVer 2.0.0. + /// + /// The catalog leaf. + /// True if the package is SemVer 2.0.0. + public static bool IsSemVer2(this PackageDetailsCatalogLeaf leaf) + { + var parsedPackageVersion = leaf.ParsePackageVersion(); + if (parsedPackageVersion.IsSemVer2) + { + return true; + } + + if (leaf.VerbatimVersion != null) + { + var parsedVerbatimVersion = NuGetVersion.Parse(leaf.VerbatimVersion); + if (parsedVerbatimVersion.IsSemVer2) + { + return true; + } + } + + if (leaf.DependencyGroups != null) + { + foreach (var dependencyGroup in leaf.DependencyGroups) + { + // Example: https://api.nuget.org/v3/catalog0/data/2018.10.28.07.42.42/mvcsitemapprovider.3.3.0-pre1.json + if (dependencyGroup.Dependencies == null) + { + continue; + } + + foreach (var dependency in dependencyGroup.Dependencies) + { + var versionRange = dependency.ParseRange(); + if ((versionRange.MaxVersion != null && versionRange.MaxVersion.IsSemVer2) + || (versionRange.MinVersion != null && versionRange.MinVersion.IsSemVer2)) + { + return true; + } + } + } + } + + return false; + } + } +} diff --git a/src/BaGet.Protocol/Catalog/CatalogPage.cs b/src/BaGet.Protocol/Catalog/CatalogPage.cs new file mode 100644 index 00000000..8c41ca29 --- /dev/null +++ b/src/BaGet.Protocol/Catalog/CatalogPage.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace BaGet.Protocol +{ + /// + /// A catalog page, used to discover catalog leafs. + /// Pages can be discovered from a . + /// See: https://docs.microsoft.com/en-us/nuget/api/catalog-resource#catalog-page + /// Based off: https://github.com/NuGet/NuGet.Services.Metadata/blob/64af0b59c5a79e0143f0808b39946df9f16cb2e7/src/NuGet.Protocol.Catalog/Models/CatalogPage.cs + /// + public class CatalogPage + { + [JsonProperty("commitTimeStamp")] + public DateTimeOffset CommitTimestamp { get; set; } + + [JsonProperty("count")] + public int Count { get; set; } + + [JsonProperty("items")] + public List Items { get; set; } + + [JsonProperty("parent")] + public string Parent { get; set; } + } +} diff --git a/src/BaGet.Protocol/Catalog/CatalogPageItem.cs b/src/BaGet.Protocol/Catalog/CatalogPageItem.cs new file mode 100644 index 00000000..034b2690 --- /dev/null +++ b/src/BaGet.Protocol/Catalog/CatalogPageItem.cs @@ -0,0 +1,22 @@ +using System; +using Newtonsoft.Json; + +namespace BaGet.Protocol +{ + /// + /// An item in the that references a . + /// See: https://docs.microsoft.com/en-us/nuget/api/catalog-resource#catalog-page-object-in-the-index + /// Based off: https://github.com/NuGet/NuGet.Services.Metadata/blob/64af0b59c5a79e0143f0808b39946df9f16cb2e7/src/NuGet.Protocol.Catalog/Models/CatalogPageItem.cs + /// + public class CatalogPageItem + { + [JsonProperty("@id")] + public string Url { get; set; } + + [JsonProperty("commitTimeStamp")] + public DateTimeOffset CommitTimestamp { get; set; } + + [JsonProperty("count")] + public int Count { get; set; } + } +} diff --git a/src/BaGet.Protocol/Catalog/ICatalogLeafItem.cs b/src/BaGet.Protocol/Catalog/ICatalogLeafItem.cs new file mode 100644 index 00000000..21134a76 --- /dev/null +++ b/src/BaGet.Protocol/Catalog/ICatalogLeafItem.cs @@ -0,0 +1,18 @@ +using System; + +namespace BaGet.Protocol +{ + /// + /// A catalog leaf. Represents a single package event. + /// Leafs can be discovered from a . + /// See: https://docs.microsoft.com/en-us/nuget/api/catalog-resource#catalog-leaf + /// Based off: https://github.com/NuGet/NuGet.Services.Metadata/blob/64af0b59c5a79e0143f0808b39946df9f16cb2e7/src/NuGet.Protocol.Catalog/Models/ICatalogLeafItem.cs + /// + public interface ICatalogLeafItem + { + DateTimeOffset CommitTimestamp { get; } + string PackageId { get; } + string PackageVersion { get; } + CatalogLeafType Type { get; } + } +} diff --git a/src/BaGet.Protocol/Catalog/ICatalogResource.cs b/src/BaGet.Protocol/Catalog/ICatalogResource.cs new file mode 100644 index 00000000..ed3b81bb --- /dev/null +++ b/src/BaGet.Protocol/Catalog/ICatalogResource.cs @@ -0,0 +1,54 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace BaGet.Protocol +{ + /// + /// The Catalog resource that records all package operations. + /// You can use this resource to query for all published packages. + /// See: https://docs.microsoft.com/en-us/nuget/api/catalog-resource + /// + public interface ICatalogResource + { + /// + /// Get the entry point for the catalog resource. + /// See: https://docs.microsoft.com/en-us/nuget/api/catalog-resource#catalog-index + /// + /// A token to cancel the task. + /// The catalog index. + Task GetIndexAsync(CancellationToken cancellationToken = default); + + /// + /// Get a single catalog page, used to discover catalog leafs. + /// See: https://docs.microsoft.com/en-us/nuget/api/catalog-resource#catalog-page + /// + /// The URL of the page, from the . + /// A token to cancel the task. + /// A catalog page. + Task GetPageAsync( + string pageUrl, + CancellationToken cancellationToken = default); + + /// + /// Get a single catalog leaf, representing a package deletion event. + /// See: https://docs.microsoft.com/en-us/nuget/api/catalog-resource#catalog-leaf + /// + /// The URL of the leaf, from a . + /// A token to cancel the task. + /// A catalog leaf. + Task GetPackageDeleteLeafAsync( + string leafUrl, + CancellationToken cancellationToken = default); + + /// + /// Get a single catalog leaf, representing a package creation or update event. + /// See: https://docs.microsoft.com/en-us/nuget/api/catalog-resource#catalog-leaf + /// + /// The URL of the leaf, from a . + /// A token to cancel the task. + /// A catalog leaf. + Task GetPackageDetailsLeafAsync( + string leafUrl, + CancellationToken cancellationToken = default); + } +} diff --git a/src/BaGet.Protocol/Catalog/PackageDeleteCatalogLeaf.cs b/src/BaGet.Protocol/Catalog/PackageDeleteCatalogLeaf.cs new file mode 100644 index 00000000..3ee44b72 --- /dev/null +++ b/src/BaGet.Protocol/Catalog/PackageDeleteCatalogLeaf.cs @@ -0,0 +1,12 @@ +namespace BaGet.Protocol +{ + /// + /// A "package delete" catalog leaf. Represents a single package deletion event. + /// Leafs can be discovered from a . + /// See: https://docs.microsoft.com/en-us/nuget/api/catalog-resource#catalog-leaf + /// Based off: https://github.com/NuGet/NuGet.Services.Metadata/blob/64af0b59c5a79e0143f0808b39946df9f16cb2e7/src/NuGet.Protocol.Catalog/Models/PackageDeleteCatalogLeaf.cs + /// + public class PackageDeleteCatalogLeaf : CatalogLeaf + { + } +} diff --git a/src/BaGet.Protocol/Catalog/PackageDetailsCatalogLeaf.cs b/src/BaGet.Protocol/Catalog/PackageDetailsCatalogLeaf.cs new file mode 100644 index 00000000..b48c288e --- /dev/null +++ b/src/BaGet.Protocol/Catalog/PackageDetailsCatalogLeaf.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace BaGet.Protocol +{ + /// + /// A "package details" catalog leaf. Represents a single package create or update event. + /// Leafs can be discovered from a . + /// See: https://docs.microsoft.com/en-us/nuget/api/catalog-resource#catalog-leaf + /// Based off: https://github.com/NuGet/NuGet.Services.Metadata/blob/64af0b59c5a79e0143f0808b39946df9f16cb2e7/src/NuGet.Protocol.Catalog/Models/PackageDetailsCatalogLeaf.cs + /// + public class PackageDetailsCatalogLeaf : CatalogLeaf + { + [JsonProperty("authors")] + public string Authors { get; set; } + + [JsonProperty("copyright")] + public string Copyright { get; set; } + + [JsonProperty("created")] + public DateTimeOffset Created { get; set; } + + [JsonProperty("lastEdited")] + public DateTimeOffset LastEdited { get; set; } + + [JsonProperty("dependencyGroups")] + public List DependencyGroups { get; set; } + + [JsonProperty("description")] + public string Description { get; set; } + + [JsonProperty("iconUrl")] + public string IconUrl { get; set; } + + /// + /// Note that an old bug in the NuGet.org catalog had this wrong in some cases. + /// Example: https://api.nuget.org/v3/catalog0/data/2016.03.11.21.02.55/mvid.fody.2.json + /// + [JsonProperty("isPrerelease")] + public bool IsPrerelease { get; set; } + + [JsonProperty("language")] + public string Language { get; set; } + + [JsonProperty("licenseUrl")] + public string LicenseUrl { get; set; } + + [JsonProperty("listed")] + public bool? Listed { get; set; } + + [JsonProperty("minClientVersion")] + public string MinClientVersion { get; set; } + + [JsonProperty("packageHash")] + public string PackageHash { get; set; } + + [JsonProperty("packageHashAlgorithm")] + public string PackageHashAlgorithm { get; set; } + + [JsonProperty("packageSize")] + public long PackageSize { get; set; } + + [JsonProperty("projectUrl")] + public string ProjectUrl { get; set; } + + [JsonProperty("releaseNotes")] + public string ReleaseNotes { get; set; } + + [JsonProperty("requireLicenseAgreement")] + public bool? RequireLicenseAgreement { get; set; } + + [JsonProperty("summary")] + public string Summary { get; set; } + + [JsonProperty("tags")] + public List Tags { get; set; } + + [JsonProperty("title")] + public string Title { get; set; } + + [JsonProperty("verbatimVersion")] + public string VerbatimVersion { get; set; } + + [JsonProperty("licenseExpression")] + public string LicenseExpression { get; set; } + + [JsonProperty("licenseFile")] + public string LicenseFile { get; set; } + + [JsonProperty("iconFile")] + public string IconFile { get; set; } + } +} diff --git a/src/BaGet.Protocol/Converters/BaseCatalogLeafConverter.cs b/src/BaGet.Protocol/Converters/BaseCatalogLeafConverter.cs new file mode 100644 index 00000000..da110305 --- /dev/null +++ b/src/BaGet.Protocol/Converters/BaseCatalogLeafConverter.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace BaGet.Protocol.Internal +{ + /// + /// Based off: https://github.com/NuGet/NuGet.Services.Metadata/blob/64af0b59c5a79e0143f0808b39946df9f16cb2e7/src/NuGet.Protocol.Catalog/Serialization/BaseCatalogLeafConverter.cs + /// + internal abstract class BaseCatalogLeafConverter : JsonConverter + { + private readonly IReadOnlyDictionary _fromType; + + public BaseCatalogLeafConverter(IReadOnlyDictionary fromType) + { + _fromType = fromType; + } + + public override bool CanConvert(Type objectType) + { + return objectType == typeof(CatalogLeafType); + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + string output; + if (_fromType.TryGetValue((CatalogLeafType)value, out output)) + { + writer.WriteValue(output); + } + + throw new NotSupportedException($"The catalog leaf type '{value}' is not supported."); + } + } +} diff --git a/src/BaGet.Protocol/Converters/CatalogLeafItemTypeConverter.cs b/src/BaGet.Protocol/Converters/CatalogLeafItemTypeConverter.cs new file mode 100644 index 00000000..59ad8baf --- /dev/null +++ b/src/BaGet.Protocol/Converters/CatalogLeafItemTypeConverter.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Newtonsoft.Json; + +namespace BaGet.Protocol.Internal +{ + /// + /// Based off: https://github.com/NuGet/NuGet.Services.Metadata/blob/64af0b59c5a79e0143f0808b39946df9f16cb2e7/src/NuGet.Protocol.Catalog/Serialization/CatalogLeafItemTypeConverter.cs + /// + internal class CatalogLeafItemTypeConverter : BaseCatalogLeafConverter + { + private static readonly Dictionary FromType = new Dictionary + { + { CatalogLeafType.PackageDelete, "nuget:PackageDelete" }, + { CatalogLeafType.PackageDetails, "nuget:PackageDetails" }, + }; + + private static readonly Dictionary FromString = FromType + .ToDictionary(x => x.Value, x => x.Key); + + public CatalogLeafItemTypeConverter() : base(FromType) + { + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + var stringValue = reader.Value as string; + if (stringValue != null) + { + CatalogLeafType output; + if (FromString.TryGetValue(stringValue, out output)) + { + return output; + } + } + + throw new JsonSerializationException($"Unexpected value for a {nameof(CatalogLeafType)}."); + } + } +} diff --git a/src/BaGet.Protocol/Converters/CatalogLeafTypeConverter.cs b/src/BaGet.Protocol/Converters/CatalogLeafTypeConverter.cs new file mode 100644 index 00000000..aec89a08 --- /dev/null +++ b/src/BaGet.Protocol/Converters/CatalogLeafTypeConverter.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Newtonsoft.Json; + +namespace BaGet.Protocol.Internal +{ + /// + /// Based off: https://github.com/NuGet/NuGet.Services.Metadata/blob/64af0b59c5a79e0143f0808b39946df9f16cb2e7/src/NuGet.Protocol.Catalog/Serialization/CatalogLeafTypeConverter.cs + /// + internal class CatalogLeafTypeConverter : BaseCatalogLeafConverter + { + private static readonly Dictionary FromType = new Dictionary + { + { CatalogLeafType.PackageDelete, "PackageDelete" }, + { CatalogLeafType.PackageDetails, "PackageDetails" }, + }; + + private static readonly Dictionary FromString = FromType + .ToDictionary(x => x.Value, x => x.Key); + + public CatalogLeafTypeConverter() : base(FromType) + { + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + List types; + if (reader.TokenType == JsonToken.StartArray) + { + types = serializer.Deserialize>(reader); + } + else + { + types = new List { reader.Value }; + } + + foreach (var type in types.OfType()) + { + CatalogLeafType output; + if (FromString.TryGetValue(type, out output)) + { + return output; + } + } + + throw new JsonSerializationException($"Unexpected value for a {nameof(CatalogLeafType)}."); + } + } +} diff --git a/src/BaGet.Protocol/Converters/PackageDependencyRangeConverter.cs b/src/BaGet.Protocol/Converters/PackageDependencyRangeConverter.cs new file mode 100644 index 00000000..2e988238 --- /dev/null +++ b/src/BaGet.Protocol/Converters/PackageDependencyRangeConverter.cs @@ -0,0 +1,33 @@ +using System; +using System.Linq; +using Newtonsoft.Json; + +namespace BaGet.Protocol.Internal +{ + public class PackageDependencyRangeConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return objectType == typeof(string); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + if (reader.TokenType == JsonToken.StartArray) + { + // There are some quirky packages with arrays of dependency version ranges. In this case, we take the + // first element. + // Example: https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json + var array = serializer.Deserialize(reader); + return array.FirstOrDefault(); + } + + return serializer.Deserialize(reader); + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + serializer.Serialize(writer, value); + } + } +} diff --git a/src/BaGet.Protocol/INuGetClientFactory.cs b/src/BaGet.Protocol/INuGetClientFactory.cs index 4f681cbd..3eb0c663 100644 --- a/src/BaGet.Protocol/INuGetClientFactory.cs +++ b/src/BaGet.Protocol/INuGetClientFactory.cs @@ -36,5 +36,12 @@ public interface INuGetClientFactory /// /// A client to interact with the NuGet Search resource. Task CreateSearchClientAsync(CancellationToken cancellationToken = default); + + /// + /// Create a low level client to interact with the NuGet catalog resource. + /// See: https://docs.microsoft.com/en-us/nuget/api/catalog-resource + /// + /// A client to interact with the Catalog resource. + Task CreateCatalogClientAsync(CancellationToken cancellationToken = default); } } diff --git a/src/BaGet.Protocol/NuGetClientFactory.cs b/src/BaGet.Protocol/NuGetClientFactory.cs index 164a14f0..feb10652 100644 --- a/src/BaGet.Protocol/NuGetClientFactory.cs +++ b/src/BaGet.Protocol/NuGetClientFactory.cs @@ -23,6 +23,7 @@ public class NuGetClientFactory : INuGetClientFactory private static readonly string Version470 = "/4.7.0"; private static readonly string Version490 = "/4.9.0"; + private static readonly string[] Catalog = { "Catalog" + Version300 }; private static readonly string[] SearchQueryService = { "SearchQueryService" + Versioned, "SearchQueryService" + Version340, "SearchQueryService" + Version300beta }; private static readonly string[] RegistrationsBaseUrl = { "RegistrationsBaseUrl" + Versioned, "RegistrationsBaseUrl" + Version360, "RegistrationsBaseUrl" + Version340, "RegistrationsBaseUrl" + Version300beta }; private static readonly string[] SearchAutocompleteService = { "SearchAutocompleteService" + Versioned, "SearchAutocompleteService" + Version300beta }; @@ -41,7 +42,6 @@ public class NuGetClientFactory : INuGetClientFactory public NuGetClientFactory(HttpClient httpClient, string serviceIndexUrl) { - // TODO: Integrate with HttpClientFactory? _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); _serviceIndexUrl = serviceIndexUrl ?? throw new ArgumentNullException(nameof(serviceIndexUrl)); @@ -91,6 +91,18 @@ public Task CreateSearchClientAsync(CancellationToken cancellat return GetClientAsync(c => c.SearchClient, cancellationToken); } + /// + /// Create a low level client to interact with the NuGet catalog resource. + /// See: https://docs.microsoft.com/en-us/nuget/api/catalog-resource + /// + /// A client to interact with the Catalog resource. + public Task CreateCatalogClientAsync(CancellationToken cancellationToken = default) + { + // TODO: There are multiple search endpoints to support high read availability. + // This factory should create a search client that uses all these endpoints. + return GetClientAsync(c => c.CatalogClient, cancellationToken); + } + private async Task GetClientAsync(Func clientFactory, CancellationToken cancellationToken) { // TODO: This should periodically refresh the service index response. @@ -108,6 +120,7 @@ private async Task GetClientAsync(Func clientFactory, Can var contentClient = new PackageContentClient(_httpClient, GetResourceUrl(serviceIndex, PackageBaseAddress)); var metadataClient = new PackageMetadataClient(_httpClient, GetResourceUrl(serviceIndex, RegistrationsBaseUrl)); + var catalogClient = new CatalogClient(_httpClient, GetResourceUrl(serviceIndex, Catalog)); var searchClient = new SearchClient( _httpClient, GetResourceUrl(serviceIndex, SearchQueryService), @@ -119,6 +132,7 @@ private async Task GetClientAsync(Func clientFactory, Can PackageContentClient = contentClient, PackageMetadataClient = metadataClient, SearchClient = searchClient, + CatalogClient = catalogClient, }; } } @@ -144,6 +158,7 @@ private class NuGetClients public IPackageContentResource PackageContentClient { get; set; } public IPackageMetadataResource PackageMetadataClient { get; set; } public ISearchResource SearchClient { get; set; } + public ICatalogResource CatalogClient { get; set; } } } } diff --git a/src/BaGet.Protocol/PackageMetadata/DependencyGroupItem.cs b/src/BaGet.Protocol/PackageMetadata/DependencyGroupItem.cs new file mode 100644 index 00000000..97a5f691 --- /dev/null +++ b/src/BaGet.Protocol/PackageMetadata/DependencyGroupItem.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace BaGet.Protocol +{ + /// + /// The dependencies of the package for a specific target framework. + /// See: https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#package-dependency-group + /// + public class DependencyGroupItem + { + [JsonProperty("targetFramework")] + public string TargetFramework { get; set; } + + [JsonProperty("dependencies", DefaultValueHandling = DefaultValueHandling.Ignore)] + public List Dependencies { get; set; } + } +} diff --git a/src/BaGet.Protocol/PackageMetadata/DependencyItem.cs b/src/BaGet.Protocol/PackageMetadata/DependencyItem.cs new file mode 100644 index 00000000..93ebe272 --- /dev/null +++ b/src/BaGet.Protocol/PackageMetadata/DependencyItem.cs @@ -0,0 +1,19 @@ +using BaGet.Protocol.Internal; +using Newtonsoft.Json; + +namespace BaGet.Protocol +{ + /// + /// Represents a package dependency. + /// See: https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#package-dependency + /// + public class DependencyItem + { + [JsonProperty("id")] + public string Id { get; set; } + + [JsonProperty("range")] + [JsonConverter(typeof(PackageDependencyRangeConverter))] + public string Range { get; set; } + } +} diff --git a/src/BaGet.Protocol/PackageMetadata/PackageMetadata.cs b/src/BaGet.Protocol/PackageMetadata/PackageMetadata.cs index 640688e6..8c141152 100644 --- a/src/BaGet.Protocol/PackageMetadata/PackageMetadata.cs +++ b/src/BaGet.Protocol/PackageMetadata/PackageMetadata.cs @@ -81,53 +81,4 @@ public PackageMetadata( public string Title { get; } public IReadOnlyList DependencyGroups { get; } } - - /// - /// The dependencies of the package for a specific target framework. - /// See: https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#package-dependency-group - /// - public class DependencyGroupItem - { - public DependencyGroupItem( - string catalogUri, - string targetFramework, - IReadOnlyList dependencies) - { - CatalogUri = catalogUri; - Type = "PackageDependencyGroup"; - TargetFramework = targetFramework; - Dependencies = (dependencies?.Count > 0) ? dependencies : null; - } - - [JsonProperty(PropertyName = "@id")] - public string CatalogUri { get; } - - [JsonProperty(PropertyName = "@type")] - public string Type { get; } - - public string TargetFramework { get; } - - [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] - public IReadOnlyList Dependencies { get; } - } - - public class DependencyItem - { - [JsonProperty(PropertyName = "@id")] - public string DepId { get; } - - [JsonProperty(PropertyName = "@type")] - public string Type { get; } - - public string Id { get; } - public string Range { get; } - - public DependencyItem(string depId, string id, string range) - { - DepId = depId; - Type = "PackageDependency"; - Id = id; - Range = range; - } - } } diff --git a/tests/BaGet.Protocol.Tests/CatalogClientTests.cs b/tests/BaGet.Protocol.Tests/CatalogClientTests.cs new file mode 100644 index 00000000..c21a3660 --- /dev/null +++ b/tests/BaGet.Protocol.Tests/CatalogClientTests.cs @@ -0,0 +1,55 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using BaGet.Protocol.Internal; +using Xunit; + +namespace BaGet.Protocol.Tests +{ + public class CatalogClientTests : IClassFixture + { + private readonly CatalogClient _target; + + public CatalogClientTests(ProtocolFixture fixture) + { + _target = fixture.CatalogClient; + } + + [Fact] + public async Task GetsCatalogIndex() + { + var result = await _target.GetIndexAsync(); + + Assert.NotNull(result); + Assert.True(result.Count > 0); + Assert.NotEmpty(result.Items); + } + + [Fact] + public async Task GetsCatalogLeaf() + { + var index = await _target.GetIndexAsync(); + var pageItem = index.Items.First(); + + var page = await _target.GetPageAsync(pageItem.Url); + var leafItem = page.Items.First(); + + CatalogLeaf result; + switch (leafItem.Type) + { + case CatalogLeafType.PackageDelete: + result = await _target.GetPackageDeleteLeafAsync(leafItem.Url); + break; + + case CatalogLeafType.PackageDetails: + result = await _target.GetPackageDetailsLeafAsync(leafItem.Url); + break; + + default: + throw new NotSupportedException($"Unknown leaf type '{leafItem.Type}'"); + } + + Assert.NotNull(result.PackageId); + } + } +} diff --git a/tests/BaGet.Protocol.Tests/Support/ProtocolFixture.cs b/tests/BaGet.Protocol.Tests/Support/ProtocolFixture.cs index 785edbe1..ff6adcb3 100644 --- a/tests/BaGet.Protocol.Tests/Support/ProtocolFixture.cs +++ b/tests/BaGet.Protocol.Tests/Support/ProtocolFixture.cs @@ -16,6 +16,7 @@ public ProtocolFixture() ServiceIndexClient = new ServiceIndexClient(httpClient, "https://api.nuget.org/v3/index.json"); ContentClient = new PackageContentClient(httpClient, "https://api.nuget.org/v3-flatcontainer"); MetadataClient = new PackageMetadataClient(httpClient, "https://api.nuget.org/v3/registration3-gz-semver2"); + CatalogClient = new CatalogClient(httpClient, "https://api.nuget.org/v3/catalog0/index.json"); SearchClient = new SearchClient( httpClient, "https://azuresearch-usnc.nuget.org/query", @@ -26,5 +27,6 @@ public ProtocolFixture() public PackageContentClient ContentClient { get; } public PackageMetadataClient MetadataClient { get; } public SearchClient SearchClient { get; } + public CatalogClient CatalogClient { get; } } } diff --git a/tests/BaGet.Tests/BaGet.Tests.csproj b/tests/BaGet.Tests/BaGet.Tests.csproj index db6eaf1e..d375c775 100644 --- a/tests/BaGet.Tests/BaGet.Tests.csproj +++ b/tests/BaGet.Tests/BaGet.Tests.csproj @@ -13,7 +13,7 @@ - +