Apply document filters to DocumentSet<T> at query time (#378)

Motivation ---------- Some forms of querying using `DocumentSet<T>` properties on an inherited `BucketContext` don't apply document filters such as the `DocumentType` attribute. Modifications ------------- - Split `CollectionQueryable<T>` into two types `CollectionQueryable<T>` and a base `CouchbaseQueryable<T>`. The former is used to represent the original extent of a query against a collection. The latter is used to construct new `IQueryable<T>` instances as the query is extended with predicates, etc. This removes unnecessary fields and logic from the simpler case that is repeated for every new LINQ method applied to the query. - Add an additional non-generic `IDocumentSet` interface which is inherited by `IDocumentSet<T>`. This is a SemVer safe change because it only adds `IQueryable` which was already included on `IDocumentSet<T>` via `IQueryable<T>`. - Create DelayedFilterQueryProvider as a wrapper for ClusterQueryProvider that applies filters to `IDocumentSet` at query execution time. - Refactor query timeouts to always be based on a callback to the `BucketContext` rather than a property, which allows a singleton `ClusterQueryExecutor` per `BucketContext` rather than per query. - Refactor `BucketContext` to build a singleton query provider, query parser, and query executor rather than recreating for every query. - Refactor `DocumentSet<T>` to get the query provider from the `BucketContext`. - Reduce interface invocations by caching the bucket, scope, and collection names once when constructing a `DocumentSet<T>` or `CollectionQueryable<T>` instead of for each property read. - Minor perf and nullable ref type improvements to `DocumentFilterSet`. - Fix some integration tests that were refering to data no longer found in the default `beer-sample` bucket. Results ------- Filters are applied consistently to `DocumentSet<T>` based on the filter configuration at the time the query is run. Queries in all cases will have fewer heap allocations and other CPU performance benefits. Resolves #376
couchbaselabs · Nov 11, 2024 · 1cb678c · 1cb678c
1 parent a999bd2
commit 1cb678c
Show file tree

Hide file tree

Showing 15 changed files with 346 additions and 190 deletions.
diff --git a/Src/Couchbase.Linq.IntegrationTests/QueryTests.cs b/Src/Couchbase.Linq.IntegrationTests/QueryTests.cs
@@ -174,7 +174,7 @@ public void Map2PocoTests_Simple_Projections_StartsWith()
             var context = new BucketContext(TestSetup.Bucket);
 
             var beers = from b in context.Query<Beer>()
-                        where b.Type == "beer" && b.Name.StartsWith("563")
+                        where b.Type == "beer" && b.Name.StartsWith("Amendment")
                         select new { name = b.Name, abv = b.Abv };
 
             var results = beers.Take(1).ToList();
@@ -1000,7 +1000,7 @@ public void SubqueryTests_ArraySubqueryContains()
             var context = new BucketContext(TestSetup.Bucket);
 
             var breweries = from brewery in context.Query<Brewery>()
-                where brewery.Type == "brewery" && brewery.Address.Contains("563 Second Street")
+                where brewery.Type == "brewery" && brewery.Address.Contains("210 Aberdeen Dr.")
                 orderby brewery.Name
                 select new {name = brewery.Name, addresses = brewery.Address};
 

diff --git a/Src/Couchbase.Linq.IntegrationTests/SingleQueryTests.cs b/Src/Couchbase.Linq.IntegrationTests/SingleQueryTests.cs
@@ -46,7 +46,7 @@ public void Single_HasResult()
             var context = new BucketContext(TestSetup.Bucket);
 
             var beers = from beer in context.Query<Beer>()
-                where beer.Name == "21A IPA"
+                where beer.Name == "Amendment Pale Ale"
                 select new {beer.Name};
 
             Console.WriteLine(beers.Single().Name);
@@ -58,8 +58,8 @@ public async Task SingleAsync_HasResult()
             var context = new BucketContext(TestSetup.Bucket);
 
             var beers = from beer in context.Query<Beer>()
-                where beer.Name == "21A IPA"
-                select new {beer.Name};
+                where beer.Name == "Amendment Pale Ale"
+                select new { beer.Name };
 
             Console.WriteLine((await beers.SingleAsync()).Name);
         }
@@ -72,7 +72,7 @@ public async Task SingleAsync_WithPredicate_HasResult()
             var beers = from beer in context.Query<Beer>()
                 select new {beer.Name};
 
-            var result = await beers.SingleAsync(p => p.Name == "21A IPA");
+            var result = await beers.SingleAsync(p => p.Name == "Amendment Pale Ale");
 
             Console.WriteLine(result.Name);
         }
@@ -137,7 +137,7 @@ public void SingleOrDefault_HasResult()
             var context = new BucketContext(TestSetup.Bucket);
 
             var beers = from beer in context.Query<Beer>()
-                where beer.Name == "21A IPA"
+                where beer.Name == "Amendment Pale Ale"
                 select new {beer.Name};
 
             var aBeer = beers.SingleOrDefault();
@@ -151,7 +151,7 @@ public async Task SingleOrDefaultAsync_HasResult()
             var context = new BucketContext(TestSetup.Bucket);
 
             var beers = from beer in context.Query<Beer>()
-                where beer.Name == "21A IPA"
+                where beer.Name == "Amendment Pale Ale"
                 select new {beer.Name};
 
             var aBeer = await beers.SingleOrDefaultAsync();
@@ -167,7 +167,7 @@ public async Task SingleOrDefaultAsync_WithPredicate_HasResult()
             var beers = from beer in context.Query<Beer>()
                 select new {beer.Name};
 
-            var aBeer = await beers.SingleOrDefaultAsync(p => p.Name == "21A IPA");
+            var aBeer = await beers.SingleOrDefaultAsync(p => p.Name == "Amendment Pale Ale");
             Assert.IsNotNull(aBeer);
             Console.WriteLine(aBeer.Name);
         }

diff --git a/Src/Couchbase.Linq.UnitTests/Metadata/ContextMetadataTests.cs b/Src/Couchbase.Linq.UnitTests/Metadata/ContextMetadataTests.cs
@@ -1,7 +1,5 @@
-using System;
-using Couchbase.Linq.Metadata;
+using Couchbase.Linq.Metadata;
 using Couchbase.Linq.UnitTests.Documents;
-using Moq;
 using NUnit.Framework;
 
 namespace Couchbase.Linq.UnitTests.Metadata
@@ -47,6 +45,10 @@ public void ctor_ValidInitializer()
 
         private class TestContext : BucketContext
         {
+            public TestContext() : base(QueryFactory.CreateMockBucket("default"))
+            {
+            }
+
             public IDocumentSet<Beer> Beers { get; set; }
 
             public IDocumentSet<RouteInCollection> Routes { get; set; }

diff --git a/Src/Couchbase.Linq.UnitTests/N1QLTestBase.cs b/Src/Couchbase.Linq.UnitTests/N1QLTestBase.cs
@@ -1,6 +1,7 @@
 using System;
 using System.Linq;
 using System.Linq.Expressions;
+using System.Security.Cryptography.X509Certificates;
 using Couchbase.Core.IO.Serializers;
 using Couchbase.Core.Version;
 using Couchbase.KeyValue;
@@ -13,6 +14,7 @@
 using Microsoft.Extensions.Logging;
 using Moq;
 using Newtonsoft.Json.Serialization;
+using Remotion.Linq;
 
 namespace Couchbase.Linq.UnitTests
 {
@@ -109,30 +111,8 @@ internal string CreateN1QlQuery(IBucket bucket, Expression expression, ClusterVe
             return visitor.GetQuery();
         }
 
-        protected virtual IQueryable<T> CreateQueryable<T>(string bucketName)
-        {
-            return CreateQueryable<T>(bucketName, QueryExecutor);
-        }
-
-        internal virtual IQueryable<T> CreateQueryable<T>(string bucketName, IAsyncQueryExecutor queryExecutor)
-        {
-            var mockCluster = new Mock<ICluster>();
-            mockCluster
-                .Setup(p => p.ClusterServices)
-                .Returns(ServiceProvider);
-
-            var mockBucket = new Mock<IBucket>();
-            mockBucket.SetupGet(e => e.Name).Returns(bucketName);
-            mockBucket.SetupGet(e => e.Cluster).Returns(mockCluster.Object);
-
-            var mockCollection = new Mock<ICouchbaseCollection>();
-            mockCollection
-                .SetupGet(p => p.Scope.Bucket)
-                .Returns(mockBucket.Object);
-
-            return new CollectionQueryable<T>(mockCollection.Object,
-                QueryParserHelper.CreateQueryParser(mockCluster.Object), queryExecutor);
-        }
+        protected virtual IQueryable<T> CreateQueryable<T>(string bucketName) =>
+            QueryFactory.Queryable<T>(bucketName, N1QlHelpers.DefaultScopeName, N1QlHelpers.DefaultCollectionName, QueryExecutor);
 
         protected void SetContractResolver(IContractResolver contractResolver)
         {

diff --git a/Src/Couchbase.Linq.UnitTests/QueryFactory.cs b/Src/Couchbase.Linq.UnitTests/QueryFactory.cs
@@ -2,31 +2,53 @@
 using Couchbase.Core.IO.Serializers;
 using Couchbase.Core.Version;
 using Couchbase.KeyValue;
+using Couchbase.Linq.Execution;
+using Couchbase.Linq.Filters;
+using Couchbase.Linq.QueryGeneration;
 using Couchbase.Linq.Serialization;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
 using Moq;
 
 namespace Couchbase.Linq.UnitTests
 {
     internal class QueryFactory
     {
         public static IQueryable<T> Queryable<T>(IBucket bucket) =>
-            Queryable<T>(bucket.Name, "_default", "_default");
+            Queryable<T>(bucket.Name, N1QlHelpers.DefaultScopeName, N1QlHelpers.DefaultCollectionName);
 
         public static IQueryable<T> Queryable<T>(IBucket bucket, string scopeName, string collectionName) =>
             Queryable<T>(bucket.Name, scopeName, collectionName);
 
         public static IQueryable<T> Queryable<T>(string bucketName) =>
-            Queryable<T>(bucketName, "_default", "_default");
+            Queryable<T>(bucketName, N1QlHelpers.DefaultScopeName, N1QlHelpers.DefaultCollectionName);
 
-        public static IQueryable<T> Queryable<T>(string bucketName, string scopeName, string collectionName)
+        public static IQueryable<T> Queryable<T>(string bucketName, string scopeName, string collectionName) =>
+            Queryable<T>(bucketName, scopeName, collectionName, Mock.Of<IAsyncQueryExecutor>());
+
+        public static IQueryable<T> Queryable<T>(string bucketName, string scopeName, string collectionName, IAsyncQueryExecutor queryExecutor)
+        {
+            var mockCollection = CreateMockCollection(bucketName, scopeName, collectionName);
+
+            return new CollectionQueryable<T>(mockCollection,
+                new ClusterQueryProvider(
+                    QueryParserHelper.CreateQueryParser(mockCollection.Scope.Bucket.Cluster),
+                    queryExecutor));
+        }
+
+        public static ICouchbaseCollection CreateMockCollection(string bucketName, string scopeName, string collectionName) =>
+            CreateMockBucket(bucketName).Scope(scopeName).Collection(collectionName);
+
+        public static IBucket CreateMockBucket(string bucketName)
         {
             var serializer = new DefaultSerializer();
 
-            var services = new ServiceCollection();
+            IServiceCollection services = new ServiceCollection();
 
             services.AddSingleton<ITypeSerializer>(serializer);
-            services.AddLogging();
+            services.AddSingleton(new DocumentFilterManager());
+            services.Add(ServiceDescriptor.Singleton(typeof(ILogger<>), typeof(NullLogger<>)));
             services.AddSingleton(Mock.Of<IClusterVersionProvider>());
             services.AddSingleton<ISerializationConverterProvider>(
                 new DefaultSerializationConverterProvider(serializer,
@@ -38,21 +60,42 @@ public static IQueryable<T> Queryable<T>(string bucketName, string scopeName, st
                 .Returns(services.BuildServiceProvider());
 
             var mockBucket = new Mock<IBucket>();
-            mockBucket.SetupGet(e => e.Name).Returns(bucketName);
-            mockBucket.SetupGet(e => e.Cluster).Returns(mockCluster.Object);
-
-            var mockCollection = new Mock<ICouchbaseCollection>();
-            mockCollection
-                .SetupGet(p => p.Scope.Bucket)
-                .Returns(mockBucket.Object);
-            mockCollection
-                .SetupGet(p => p.Scope.Name)
-                .Returns(scopeName);
-            mockCollection
-                .SetupGet(p => p.Name)
-                .Returns(collectionName);
-
-            return new CollectionQueryable<T>(mockCollection.Object, default);
+            mockBucket
+                .SetupGet(e => e.Name)
+                .Returns(bucketName);
+            mockBucket
+                .SetupGet(e => e.Cluster)
+                .Returns(mockCluster.Object);
+            mockBucket
+                .Setup(e => e.Scope(It.IsAny<string>()))
+                .Returns((string scopeName) =>
+                {
+                    var mockScope = new Mock<IScope>();
+                    mockScope
+                        .SetupGet(p => p.Name)
+                        .Returns(scopeName);
+                    mockScope
+                        .SetupGet(p => p.Bucket)
+                        .Returns(mockBucket.Object);
+                    mockScope
+                        .Setup(e => e.Collection(It.IsAny<string>()))
+                        .Returns((string collectionName) =>
+                        {
+                            var mockCollection = new Mock<ICouchbaseCollection>();
+                            mockCollection
+                                .SetupGet(p => p.Name)
+                                .Returns(collectionName);
+                            mockCollection
+                                .SetupGet(p => p.Scope)
+                                .Returns(mockScope.Object);
+
+                            return mockCollection.Object;
+                        });
+
+                    return mockScope.Object;
+                });
+
+            return mockBucket.Object;
         }
     }
 }
diff --git a/Src/Couchbase.Linq/BucketContext.cs b/Src/Couchbase.Linq/BucketContext.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Linq;
+using Couchbase.Linq.Execution;
 using Couchbase.Linq.Filters;
 using Couchbase.Linq.Metadata;
 using Couchbase.Linq.Utils;
@@ -13,23 +14,17 @@ namespace Couchbase.Linq
     public class BucketContext : IBucketContext
     {
         private readonly DocumentFilterManager _documentFilterManager;
-
-        /// <summary>
-        /// Unit testing seam only, do not use!
-        /// </summary>
-#pragma warning disable 8618
-        internal BucketContext()
-#pragma warning restore 8618
-        {
-        }
+        internal IAsyncQueryProvider QueryProvider { get; }
 
         /// <summary>
         /// Creates a new BucketContext for a given Couchbase bucket.
         /// </summary>
         /// <param name="bucket">Bucket referenced by the new BucketContext.</param>
         public BucketContext(IBucket bucket)
         {
-            Bucket = bucket ?? throw new ArgumentNullException(nameof(bucket));
+            ThrowHelpers.ThrowIfNull(bucket);
+
+            Bucket = bucket;
 
             try
             {
@@ -42,6 +37,16 @@ public BucketContext(IBucket bucket)
                     $"{nameof(DocumentFilterManager)} has not been registered with the Couchbase Cluster. Be sure {nameof(LinqClusterOptionsExtensions.AddLinq)} is called on ${nameof(ClusterOptions)} during bootstrap.");
             }
 
+            var cluster = bucket.Cluster;
+            var innerQueryProvider = new ClusterQueryProvider(
+                QueryParserHelper.CreateQueryParser(cluster),
+                new ClusterQueryExecutor(cluster)
+                {
+                    QueryTimeoutProvider = () => QueryTimeout
+                });
+
+            QueryProvider = new DelayedFilterQueryProvider(innerQueryProvider, _documentFilterManager);
+
             var myType = GetType();
             if (myType != typeof(BucketContext))
             {
@@ -70,9 +75,9 @@ public IQueryable<T> Query<T>(BucketQueryOptions options)
 
         internal IQueryable<T> Query<T>(string scope, string collection, BucketQueryOptions options = BucketQueryOptions.None)
         {
-            IQueryable<T> query = new CollectionQueryable<T>(Bucket.Scope(scope).Collection(collection), QueryTimeout);
+            IQueryable<T> query = new CollectionQueryable<T>(Bucket.Scope(scope).Collection(collection), QueryProvider);
 
-            if ((options & BucketQueryOptions.SuppressFilters) == BucketQueryOptions.None)
+            if (!options.HasFlag(BucketQueryOptions.SuppressFilters))
             {
                 query = _documentFilterManager.ApplyFilters(query);
             }