From 01b2597edcefa6aec155f6b2e657ffba631e17e7 Mon Sep 17 00:00:00 2001 From: Andy Gallagher Date: Sun, 14 Jul 2024 15:37:15 +0100 Subject: [PATCH 1/6] Add in query, category, references queries to tags --- .../contentapi/porter/graphql/Content.scala | 2 +- .../contentapi/porter/graphql/RootQuery.scala | 9 ++- .../porter/graphql/TagQueryParameters.scala | 6 +- src/main/scala/datastore/DocumentRepo.scala | 15 +++- .../scala/datastore/ElasticsearchRepo.scala | 78 ++++++++++++------- 5 files changed, 76 insertions(+), 34 deletions(-) diff --git a/src/main/scala/com/gu/contentapi/porter/graphql/Content.scala b/src/main/scala/com/gu/contentapi/porter/graphql/Content.scala index eea0d9c..44c76fb 100644 --- a/src/main/scala/com/gu/contentapi/porter/graphql/Content.scala +++ b/src/main/scala/com/gu/contentapi/porter/graphql/Content.scala @@ -147,7 +147,7 @@ object Content { ReplaceField("tags", Field("tags", OptionType(ListType(Tags.Tag)), arguments = TagQueryParameters.NonPaginatedTagQueryParameters, - resolve=ctx=> ctx.ctx.repo.tagsForList(ctx.value.tags, ctx arg TagQueryParameters.Section, ctx arg TagQueryParameters.TagType)) + resolve=ctx=> ctx.ctx.repo.tagsForList(ctx.value.tags, ctx arg TagQueryParameters.Section, ctx arg TagQueryParameters.TagType, ctx arg TagQueryParameters.Category, ctx arg TagQueryParameters.Reference)) ), ExcludeFields("atomIds", "isGone", "isExpired", "sectionId"), AddFields( diff --git a/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala b/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala index d8287ab..1358325 100644 --- a/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala +++ b/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala @@ -73,7 +73,14 @@ object RootQuery { Field("tag", TagEdge, arguments = TagQueryParameters.AllTagQueryParameters, resolve = ctx => - ctx.ctx.repo.marshalledTags(ctx arg TagQueryParameters.tagId, ctx arg TagQueryParameters.Section, ctx arg TagQueryParameters.TagType, ctx arg PaginationParameters.OrderBy, ctx arg PaginationParameters.Limit, ctx arg PaginationParameters.Cursor) + ctx.ctx.repo.marshalledTags(ctx arg TagQueryParameters.QueryString, + ctx arg TagQueryParameters.tagId, + ctx arg TagQueryParameters.Section, + ctx arg TagQueryParameters.TagType, + ctx arg TagQueryParameters.Category, + ctx arg TagQueryParameters.Reference, + ctx arg PaginationParameters.OrderBy, + ctx arg PaginationParameters.Limit, ctx arg PaginationParameters.Cursor) ), Field("atom", AtomEdge, arguments = AtomQueryParameters.AllParameters, diff --git a/src/main/scala/com/gu/contentapi/porter/graphql/TagQueryParameters.scala b/src/main/scala/com/gu/contentapi/porter/graphql/TagQueryParameters.scala index ed60340..cd17ec8 100644 --- a/src/main/scala/com/gu/contentapi/porter/graphql/TagQueryParameters.scala +++ b/src/main/scala/com/gu/contentapi/porter/graphql/TagQueryParameters.scala @@ -99,7 +99,11 @@ object TagQueryParameters { val tagId = Argument("tagId", OptionInputType(StringType), description = "Retrieve this specific tag") val Section = Argument("section", OptionInputType(StringType), description = "Only return tags from this section") val TagType = Argument("type", OptionInputType(TagTypes), description = "Type of the tag to return") - val AllTagQueryParameters = tagId :: Section :: TagType :: Cursor :: OrderBy :: Limit :: Nil + val QueryString = Argument("q", OptionInputType(StringType), description = "Generic Lucene query string for finding tags") + val Category = Argument("category", OptionInputType(StringType), description = "A category to match against tags") + val Reference = Argument("reference", OptionInputType(StringType), description = "A reference to match against tags") + val AllTagQueryParameters = QueryString :: tagId :: Section :: TagType :: Category :: + Reference :: Cursor :: OrderBy :: Limit :: Nil val NonPaginatedTagQueryParameters = Section :: TagType :: Nil } diff --git a/src/main/scala/datastore/DocumentRepo.scala b/src/main/scala/datastore/DocumentRepo.scala index 19a74ee..14d545c 100644 --- a/src/main/scala/datastore/DocumentRepo.scala +++ b/src/main/scala/datastore/DocumentRepo.scala @@ -20,10 +20,17 @@ trait DocumentRepo { orderDate:Option[String], orderBy:Option[SortOrder], limit: Option[Int], cursor: Option[String]): Future[Edge[Content]] - def marshalledTags(maybeTagId:Option[String], maybeSection: Option[String], tagType:Option[String], - orderBy: Option[SortOrder], limit: Option[Int], cursor: Option[String]): Future[Edge[Tag]] - - def tagsForList(tagIdList:Seq[String], maybeSection: Option[String], tagType:Option[String]):Future[Seq[Tag]] + def marshalledTags(maybeQuery:Option[String], + maybeTagId:Option[String], + maybeSection: Option[String], + tagType:Option[String], + maybeCategory:Option[String], + maybeReferences:Option[String], + orderBy: Option[SortOrder], + limit: Option[Int], + cursor: Option[String]): Future[Edge[Tag]] + + def tagsForList(tagIdList:Seq[String], maybeSection: Option[String], tagType:Option[String], maybeCategory:Option[String], maybeReferences:Option[String]):Future[Seq[Tag]] def atomsForList(atomIds: Seq[String], atomType: Option[String]):Future[Seq[Json]] diff --git a/src/main/scala/datastore/ElasticsearchRepo.scala b/src/main/scala/datastore/ElasticsearchRepo.scala index eecc8a4..beeb488 100644 --- a/src/main/scala/datastore/ElasticsearchRepo.scala +++ b/src/main/scala/datastore/ElasticsearchRepo.scala @@ -183,7 +183,9 @@ class ElasticsearchRepo(endpoint:ElasticNodeEndpoint, val defaultPageSize:Int=20 } } - private def tagQueryParams(maybeTagId:Option[String], maybeSection:Option[String], tagType:Option[String]):Seq[Query] = { + private def tagQueryParams(maybeTagId:Option[String], maybeSection:Option[String], + tagType:Option[String], maybeCategory:Option[String], + maybeReferences: Option[String]):Seq[Query] = { Seq( maybeTagId.map(MatchQuery("id", _)), maybeSection.map(MatchQuery("sectionId", _)), @@ -192,18 +194,28 @@ class ElasticsearchRepo(endpoint:ElasticNodeEndpoint, val defaultPageSize:Int=20 ExistsQuery("podcast") case tp: String => MatchQuery("type", tp) - }) + }), + maybeSection.map(s=>termQuery("section",s)), + maybeCategory.map(cat=>termQuery("tagCategories", cat)), + maybeReferences.map(ref=>termQuery("references", ref)) //this is an object field - check how terming works!! ).collect({ case Some(param) => param }) } - private def buildTagQuery(maybeTagId:Option[String], maybeSection:Option[String], tagType:Option[String]) = { - val base = search("tag") - val params = tagQueryParams(maybeTagId, maybeSection, tagType) + private def buildTagQuery(maybeTagId:Option[String], + maybeSection:Option[String], + tagType:Option[String], maybeQuery:Option[String], + maybeCategory:Option[String], maybeReferences:Option[String]) = { + val baseSearch = search("tag") + val searchWithQuery = maybeQuery match { + case Some(q)=>baseSearch.query(q) + case None=>baseSearch + } + val params = tagQueryParams(maybeTagId, maybeSection, tagType, maybeCategory, maybeReferences) if(params.isEmpty) { - base + searchWithQuery } else { - base.query(BoolQuery(must=params)) + searchWithQuery.query(BoolQuery(must=params)) } } @@ -217,7 +229,15 @@ class ElasticsearchRepo(endpoint:ElasticNodeEndpoint, val defaultPageSize:Int=20 //FIXME: tagsForList / marshalledTags could be DRY'd out a bit - override def marshalledTags(maybeTagId:Option[String], maybeSection: Option[String], tagType:Option[String], orderBy: Option[SortOrder], limit: Option[Int], cursor: Option[String]): Future[Edge[Tag]] = { + override def marshalledTags(maybeQuery:Option[String], + maybeTagId:Option[String], + maybeSection: Option[String], + tagType:Option[String], + maybeCategory:Option[String], + maybeReferences:Option[String], + orderBy: Option[SortOrder], + limit: Option[Int], + cursor: Option[String]): Future[Edge[Tag]] = { val pageSize = limit.getOrElse(defaultPageSize) val sortParam = if(maybeSection.isDefined || tagType.isDefined) { @@ -229,7 +249,7 @@ class ElasticsearchRepo(endpoint:ElasticNodeEndpoint, val defaultPageSize:Int=20 Edge.decodeCursor(cursor) match { case Right(maybeCursor)=> client.execute { - buildTagQuery(maybeTagId, maybeSection, tagType) + buildTagQuery(maybeTagId, maybeSection, tagType, maybeQuery, maybeCategory, maybeReferences) .sortBy(sortParam) .limit(pageSize) .searchAfter(maybeCursor) @@ -239,25 +259,10 @@ class ElasticsearchRepo(endpoint:ElasticNodeEndpoint, val defaultPageSize:Int=20 } } - override def tagsForList(tagIdList:Seq[String], maybeSection: Option[String], tagType:Option[String]):Future[Seq[Tag]] = { - val tagIdMatches = tagIdList.map(MatchQuery("id", _)) - - client.execute { - val restrictions = tagQueryParams(None, maybeSection, tagType) - - if(restrictions.nonEmpty) { - search("tag").query( - BoolQuery( - must=restrictions :+ BoolQuery(should=tagIdMatches) - ) - ) - } else { - search("tag").query(BoolQuery(should=tagIdMatches)) - } - - } flatMap { response=> + private def marshalTags(response:Future[Response[SearchResponse]]):Future[Seq[Tag]] = { + response flatMap { response=> if(response.isError) { - logger.error(s"Could not make query for tags ${tagIdList}: ${response.error}") + logger.error(s"Could not make query for tags: ${response.error}") Future.failed(response.error.asException) } else { Future(response.result.hits.hits.map(hit=> @@ -272,6 +277,25 @@ class ElasticsearchRepo(endpoint:ElasticNodeEndpoint, val defaultPageSize:Int=20 } } + override def tagsForList(tagIdList:Seq[String], maybeSection: Option[String], tagType:Option[String], maybeCategory:Option[String], maybeReferences:Option[String]):Future[Seq[Tag]] = { + val tagIdMatches = tagIdList.map(MatchQuery("id", _)) + + val response = client.execute { + val restrictions = tagQueryParams(None, maybeSection, tagType, maybeCategory, maybeReferences) + + if(restrictions.nonEmpty) { + search("tag").query( + BoolQuery( + must=restrictions :+ BoolQuery(should=tagIdMatches) + ) + ) + } else { + search("tag").query(BoolQuery(should=tagIdMatches)) + } + } + marshalTags(response) + } + private def findAtoms(atomIds: Option[Seq[String]], queryString: Option[String], queryFields: Option[Seq[String]], atomType: Option[String], revisionBefore: Option[Long], revisionAfter: Option[Long], From bd1e54b59b1b5e44cf300e77a1044b8f74edf3ff Mon Sep 17 00:00:00 2001 From: Andy Gallagher Date: Sun, 14 Jul 2024 16:04:17 +0100 Subject: [PATCH 2/6] Add fuzzy-matching option for tag names --- .../gu/contentapi/porter/graphql}/Edge.scala | 4 +-- .../contentapi/porter/graphql/RootQuery.scala | 1 + .../porter/graphql/TagQueryParameters.scala | 20 ++++++++++-- src/main/scala/datastore/DocumentRepo.scala | 1 + .../scala/datastore/ElasticsearchRepo.scala | 32 ++++++++++++------- 5 files changed, 43 insertions(+), 15 deletions(-) rename src/main/scala/{deprecated/anotherschema => com/gu/contentapi/porter/graphql}/Edge.scala (95%) diff --git a/src/main/scala/deprecated/anotherschema/Edge.scala b/src/main/scala/com/gu/contentapi/porter/graphql/Edge.scala similarity index 95% rename from src/main/scala/deprecated/anotherschema/Edge.scala rename to src/main/scala/com/gu/contentapi/porter/graphql/Edge.scala index 48ec3ec..b9f6ca5 100644 --- a/src/main/scala/deprecated/anotherschema/Edge.scala +++ b/src/main/scala/com/gu/contentapi/porter/graphql/Edge.scala @@ -9,11 +9,11 @@ import java.nio.charset.StandardCharsets import java.util.Base64 import scala.util.Try import io.circe.syntax._ -@deprecated("you should be using com.gu.contentapi.porter.graphql") + case class Edge[T:io.circe.Decoder](totalCount:Long, endCursor:Option[String], hasNextPage:Boolean, nodes:Seq[T]) { def map[V:io.circe.Decoder](mapper:(T)=>V) = Edge[V](totalCount, endCursor, hasNextPage, nodes.map(mapper)) } -@deprecated("you should be using com.gu.contentapi.porter.graphql") + object Edge { private val logger = LoggerFactory.getLogger(getClass) private val encoder = Base64.getEncoder diff --git a/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala b/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala index 1358325..84f3860 100644 --- a/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala +++ b/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala @@ -74,6 +74,7 @@ object RootQuery { arguments = TagQueryParameters.AllTagQueryParameters, resolve = ctx => ctx.ctx.repo.marshalledTags(ctx arg TagQueryParameters.QueryString, + ctx arg TagQueryParameters.Fuzziness, ctx arg TagQueryParameters.tagId, ctx arg TagQueryParameters.Section, ctx arg TagQueryParameters.TagType, diff --git a/src/main/scala/com/gu/contentapi/porter/graphql/TagQueryParameters.scala b/src/main/scala/com/gu/contentapi/porter/graphql/TagQueryParameters.scala index cd17ec8..2c3d86d 100644 --- a/src/main/scala/com/gu/contentapi/porter/graphql/TagQueryParameters.scala +++ b/src/main/scala/com/gu/contentapi/porter/graphql/TagQueryParameters.scala @@ -96,13 +96,29 @@ object TagQueryParameters { ) ) + val FuzzinessOptions = EnumType( + "FuzzinessOptions", + Some("Valid options for making a fuzzy-match query"), + List( + EnumValue("AUTO", + value="AUTO", + description=Some("Generates an edit distance based on the length of the term. If the term is >5 chars, then 2 edits allowed; if <3 chars than no edits allowed") + ), + EnumValue("OFF", + value="OFF", + description=Some("Disable fuzzy-matching") + ) + ) + ) + val tagId = Argument("tagId", OptionInputType(StringType), description = "Retrieve this specific tag") val Section = Argument("section", OptionInputType(StringType), description = "Only return tags from this section") val TagType = Argument("type", OptionInputType(TagTypes), description = "Type of the tag to return") - val QueryString = Argument("q", OptionInputType(StringType), description = "Generic Lucene query string for finding tags") + val QueryString = Argument("q", OptionInputType(StringType), description = "Search for tags that match this public-facing name") + val Fuzziness = Argument("fuzzy", OptionInputType(FuzzinessOptions), description = "Perform a fuzzy-matching query (default). Set to `OFF` to disable fuzzy-matching.") val Category = Argument("category", OptionInputType(StringType), description = "A category to match against tags") val Reference = Argument("reference", OptionInputType(StringType), description = "A reference to match against tags") - val AllTagQueryParameters = QueryString :: tagId :: Section :: TagType :: Category :: + val AllTagQueryParameters = QueryString :: tagId :: Section :: TagType :: Fuzziness :: Category :: Reference :: Cursor :: OrderBy :: Limit :: Nil val NonPaginatedTagQueryParameters = Section :: TagType :: Nil diff --git a/src/main/scala/datastore/DocumentRepo.scala b/src/main/scala/datastore/DocumentRepo.scala index 14d545c..01325b7 100644 --- a/src/main/scala/datastore/DocumentRepo.scala +++ b/src/main/scala/datastore/DocumentRepo.scala @@ -21,6 +21,7 @@ trait DocumentRepo { limit: Option[Int], cursor: Option[String]): Future[Edge[Content]] def marshalledTags(maybeQuery:Option[String], + maybeFuzziness:Option[String], maybeTagId:Option[String], maybeSection: Option[String], tagType:Option[String], diff --git a/src/main/scala/datastore/ElasticsearchRepo.scala b/src/main/scala/datastore/ElasticsearchRepo.scala index beeb488..a23d3cd 100644 --- a/src/main/scala/datastore/ElasticsearchRepo.scala +++ b/src/main/scala/datastore/ElasticsearchRepo.scala @@ -8,7 +8,7 @@ import com.sksamuel.elastic4s.ElasticDsl._ import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future import com.sksamuel.elastic4s.requests.searches.SearchResponse -import com.sksamuel.elastic4s.requests.searches.queries.{ExistsQuery, NestedQuery, Query, RangeQuery} +import com.sksamuel.elastic4s.requests.searches.queries.{DisMaxQuery, ExistsQuery, Fuzzy, FuzzyQuery, NestedQuery, Query, RangeQuery} import com.sksamuel.elastic4s.requests.searches.queries.compound.BoolQuery import com.sksamuel.elastic4s.requests.searches.queries.matches.{FieldWithOptionalBoost, MatchAllQuery, MatchQuery, MultiMatchQuery} import com.sksamuel.elastic4s.requests.searches.sort.{FieldSort, ScoreSort, Sort, SortOrder} @@ -185,8 +185,19 @@ class ElasticsearchRepo(endpoint:ElasticNodeEndpoint, val defaultPageSize:Int=20 private def tagQueryParams(maybeTagId:Option[String], maybeSection:Option[String], tagType:Option[String], maybeCategory:Option[String], - maybeReferences: Option[String]):Seq[Query] = { + maybeReferences: Option[String], queryString:Option[String], fuzziness:Option[String]):Seq[Query] = { Seq( + queryString.map(qs=>{ + if(fuzziness.getOrElse("AUTO") != "OFF") { + //Why DisMax here? Because we want to include exact-matches as well, if they are relevant. E.g. FuzzyQuery on "politics" returns no results! + DisMaxQuery(Seq( + FuzzyQuery("webTitle", qs, fuzziness), + MatchQuery("webTitle", qs) + )) + } else { + MatchQuery("webTitle", qs) + } + }), maybeTagId.map(MatchQuery("id", _)), maybeSection.map(MatchQuery("sectionId", _)), tagType.map({ @@ -204,18 +215,16 @@ class ElasticsearchRepo(endpoint:ElasticNodeEndpoint, val defaultPageSize:Int=20 private def buildTagQuery(maybeTagId:Option[String], maybeSection:Option[String], tagType:Option[String], maybeQuery:Option[String], + maybeFuzziness:Option[String], maybeCategory:Option[String], maybeReferences:Option[String]) = { val baseSearch = search("tag") - val searchWithQuery = maybeQuery match { - case Some(q)=>baseSearch.query(q) - case None=>baseSearch - } - val params = tagQueryParams(maybeTagId, maybeSection, tagType, maybeCategory, maybeReferences) + + val params = tagQueryParams(maybeTagId, maybeSection, tagType, maybeCategory, maybeReferences, maybeQuery, maybeFuzziness) if(params.isEmpty) { - searchWithQuery + baseSearch } else { - searchWithQuery.query(BoolQuery(must=params)) + baseSearch.query(BoolQuery(must=params)) } } @@ -230,6 +239,7 @@ class ElasticsearchRepo(endpoint:ElasticNodeEndpoint, val defaultPageSize:Int=20 //FIXME: tagsForList / marshalledTags could be DRY'd out a bit override def marshalledTags(maybeQuery:Option[String], + maybeFuzziness:Option[String], maybeTagId:Option[String], maybeSection: Option[String], tagType:Option[String], @@ -249,7 +259,7 @@ class ElasticsearchRepo(endpoint:ElasticNodeEndpoint, val defaultPageSize:Int=20 Edge.decodeCursor(cursor) match { case Right(maybeCursor)=> client.execute { - buildTagQuery(maybeTagId, maybeSection, tagType, maybeQuery, maybeCategory, maybeReferences) + buildTagQuery(maybeTagId, maybeSection, tagType, maybeQuery, maybeFuzziness, maybeCategory, maybeReferences) .sortBy(sortParam) .limit(pageSize) .searchAfter(maybeCursor) @@ -281,7 +291,7 @@ class ElasticsearchRepo(endpoint:ElasticNodeEndpoint, val defaultPageSize:Int=20 val tagIdMatches = tagIdList.map(MatchQuery("id", _)) val response = client.execute { - val restrictions = tagQueryParams(None, maybeSection, tagType, maybeCategory, maybeReferences) + val restrictions = tagQueryParams(None, maybeSection, tagType, maybeCategory, maybeReferences, None, None) if(restrictions.nonEmpty) { search("tag").query( From 53bd6b0bd9de89853f37f24fd9313be547b91043 Mon Sep 17 00:00:00 2001 From: Andy Gallagher Date: Sun, 14 Jul 2024 16:33:26 +0100 Subject: [PATCH 3/6] Add in a sub-query to get content matching the tags --- .../contentapi/porter/graphql/RootQuery.scala | 25 ++++++++++++++++--- .../porter/graphql/TagQueryParameters.scala | 3 ++- .../scala/datastore/ElasticsearchRepo.scala | 2 +- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala b/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala index 84f3860..437e49e 100644 --- a/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala +++ b/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala @@ -1,6 +1,7 @@ package com.gu.contentapi.porter.graphql import com.gu.contentapi.porter.model.{Content, Tag} +import com.sksamuel.elastic4s.requests.searches.sort.SortOrder import sangria.schema._ import datastore.GQLQueryContext import deprecated.anotherschema.Edge @@ -24,14 +25,32 @@ object RootQuery { ) ) - val TagEdge: ObjectType[Unit, Edge[Tag]] = ObjectType( + val TagEdge: ObjectType[GQLQueryContext, Edge[Tag]] = ObjectType( "TagEdge", "A list of tags with pagination features", - () => fields[Unit, Edge[Tag]]( + () => fields[GQLQueryContext, Edge[Tag]]( Field("totalCount", LongType, Some("Total number of results that match your query"), resolve = _.value.totalCount), Field("endCursor", OptionType(StringType), Some("The last record cursor in the set"), resolve = _.value.endCursor), Field("hasNextPage", BooleanType, Some("Whether there are any more records to retrieve"), resolve = _.value.hasNextPage), - Field("nodes", ListType(com.gu.contentapi.porter.graphql.Tags.Tag), Some("The actual tags returned"), resolve = _.value.nodes) + Field("nodes", ListType(com.gu.contentapi.porter.graphql.Tags.Tag), Some("The actual tags returned"), resolve = _.value.nodes), + Field("matching_content", ArticleEdge, Some("Content which matches any of the tags returned"), + arguments= ContentQueryParameters.AllContentQueryParameters, + resolve = { ctx=> + ctx.ctx.repo.marshalledDocs(ctx arg ContentQueryParameters.QueryString, + queryFields=ctx arg ContentQueryParameters.QueryFields, + atomId = None, + forChannel = ctx arg ContentQueryParameters.ChannelArg, + userTier = ctx.ctx.userTier, + tagIds = Some(ctx.value.nodes.map(_.id)), + excludeTags = ctx arg ContentQueryParameters.ExcludeTagArg, + sectionIds = ctx arg ContentQueryParameters.SectionArg, + excludeSections = ctx arg ContentQueryParameters.ExcludeSectionArg, + orderDate = ctx arg PaginationParameters.OrderDate, + orderBy = ctx arg PaginationParameters.OrderBy, + limit = ctx arg PaginationParameters.Limit, + cursor = ctx arg PaginationParameters.Cursor, + ) + }) ) ) diff --git a/src/main/scala/com/gu/contentapi/porter/graphql/TagQueryParameters.scala b/src/main/scala/com/gu/contentapi/porter/graphql/TagQueryParameters.scala index 2c3d86d..80df28f 100644 --- a/src/main/scala/com/gu/contentapi/porter/graphql/TagQueryParameters.scala +++ b/src/main/scala/com/gu/contentapi/porter/graphql/TagQueryParameters.scala @@ -121,5 +121,6 @@ object TagQueryParameters { val AllTagQueryParameters = QueryString :: tagId :: Section :: TagType :: Fuzziness :: Category :: Reference :: Cursor :: OrderBy :: Limit :: Nil - val NonPaginatedTagQueryParameters = Section :: TagType :: Nil + val NonPaginatedTagQueryParameters = QueryString :: tagId :: Section :: TagType :: Fuzziness :: Category :: + Reference :: Nil } diff --git a/src/main/scala/datastore/ElasticsearchRepo.scala b/src/main/scala/datastore/ElasticsearchRepo.scala index a23d3cd..ed5d00e 100644 --- a/src/main/scala/datastore/ElasticsearchRepo.scala +++ b/src/main/scala/datastore/ElasticsearchRepo.scala @@ -163,7 +163,7 @@ class ElasticsearchRepo(endpoint:ElasticNodeEndpoint, val defaultPageSize:Int=20 Some(limitToChannelQuery(selectedChannel)), queryString.map(MultiMatchQuery(_, fields = fieldsToQuery)), atomId.map(MatchQuery("atomIds.id", _)), - tagIds.map(tags=>BoolQuery(must=tags.map(MatchQuery("tags", _)))) , + tagIds.map(tags=>BoolQuery(should=tags.map(MatchQuery("tags", _)))) , excludeTags.map(tags=>BoolQuery(not=Seq(BoolQuery(should=tags.map(MatchQuery("tags", _)))))), sectionIds.map(s=>BoolQuery(should=s.map(MatchQuery("sectionId", _)))), excludeSections.map(s=>BoolQuery(not=Seq(BoolQuery(should=s.map(MatchQuery("sectionId", _)))))) From 2ca11de56df8219695236971a31413483aa74d30 Mon Sep 17 00:00:00 2001 From: Andy Gallagher Date: Mon, 15 Jul 2024 17:40:19 +0100 Subject: [PATCH 4/6] Add in a sub-query to get content matching an individual tag --- .../contentapi/porter/graphql/RootQuery.scala | 2 +- .../gu/contentapi/porter/graphql/Tags.scala | 27 ++++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala b/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala index 437e49e..9e03b8c 100644 --- a/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala +++ b/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala @@ -33,7 +33,7 @@ object RootQuery { Field("endCursor", OptionType(StringType), Some("The last record cursor in the set"), resolve = _.value.endCursor), Field("hasNextPage", BooleanType, Some("Whether there are any more records to retrieve"), resolve = _.value.hasNextPage), Field("nodes", ListType(com.gu.contentapi.porter.graphql.Tags.Tag), Some("The actual tags returned"), resolve = _.value.nodes), - Field("matching_content", ArticleEdge, Some("Content which matches any of the tags returned"), + Field("matchingAnyTag", ArticleEdge, Some("Content which matches any of the tags returned"), arguments= ContentQueryParameters.AllContentQueryParameters, resolve = { ctx=> ctx.ctx.repo.marshalledDocs(ctx arg ContentQueryParameters.QueryString, diff --git a/src/main/scala/com/gu/contentapi/porter/graphql/Tags.scala b/src/main/scala/com/gu/contentapi/porter/graphql/Tags.scala index a96b110..02cc440 100644 --- a/src/main/scala/com/gu/contentapi/porter/graphql/Tags.scala +++ b/src/main/scala/com/gu/contentapi/porter/graphql/Tags.scala @@ -7,6 +7,7 @@ import java.time.format.DateTimeFormatter import io.circe.generic.auto._ import io.circe.syntax._ import com.gu.contentapi.porter.model +import datastore.GQLQueryContext object Tags { import Content.Reference @@ -15,10 +16,30 @@ object Tags { implicit val PodcastCategory = deriveObjectType[Unit, model.PodcastCategory]() implicit val TagPodcast = deriveObjectType[Unit, model.TagPodcast]() - val Tag = deriveObjectType[Unit, model.Tag]( + val Tag = deriveObjectType[GQLQueryContext, model.Tag]( ReplaceField("type", Field("type", OptionType(TagQueryParameters.TagTypes), resolve=_.value.`type`)), - ReplaceField("alternateIds", Field("alternateIds", ListType(StringType), arguments = AlternateIdParameters.AllAlternateIdParameters, resolve= AlternateIdParameters.TagResolver[Unit])), + ReplaceField("alternateIds", Field("alternateIds", ListType(StringType), arguments = AlternateIdParameters.AllAlternateIdParameters, resolve= AlternateIdParameters.TagResolver[GQLQueryContext])), ReplaceField("tagCategories", Field("tagCategories", ListType(StringType), resolve = _.value.tagCategories.map(_.toSeq).getOrElse(Seq()))), - ReplaceField("entityIds", Field("entityIds", ListType(StringType), resolve = _.value.tagCategories.map(_.toSeq).getOrElse(Seq()))) + ReplaceField("entityIds", Field("entityIds", ListType(StringType), resolve = _.value.tagCategories.map(_.toSeq).getOrElse(Seq()))), + AddFields( + Field("matchingContent", RootQuery.ArticleEdge, description=Some("Articles and other content which have this tag"), + arguments= ContentQueryParameters.AllContentQueryParameters.filterNot(_ == ContentQueryParameters.TagArg), + resolve = { ctx=> + ctx.ctx.repo.marshalledDocs(ctx arg ContentQueryParameters.QueryString, + queryFields=ctx arg ContentQueryParameters.QueryFields, + atomId = None, + forChannel = ctx arg ContentQueryParameters.ChannelArg, + userTier = ctx.ctx.userTier, + tagIds = Some(Seq(ctx.value.id)), + excludeTags = ctx arg ContentQueryParameters.ExcludeTagArg, + sectionIds = ctx arg ContentQueryParameters.SectionArg, + excludeSections = ctx arg ContentQueryParameters.ExcludeSectionArg, + orderDate = ctx arg PaginationParameters.OrderDate, + orderBy = ctx arg PaginationParameters.OrderBy, + limit = ctx arg PaginationParameters.Limit, + cursor = ctx arg PaginationParameters.Cursor, + ) + }) + ) ) } From fed0a9f88e0e6d789e0c61aadff2958335a6b168 Mon Sep 17 00:00:00 2001 From: Andy Gallagher Date: Tue, 16 Jul 2024 17:20:41 +0100 Subject: [PATCH 5/6] Make the search cursors work by default. Add explicit sort-by-score option --- .../porter/graphql/PaginationParameters.scala | 3 +- .../contentapi/porter/graphql/RootQuery.scala | 39 ++++++++++++------- .../scala/datastore/ElasticsearchRepo.scala | 3 +- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/main/scala/com/gu/contentapi/porter/graphql/PaginationParameters.scala b/src/main/scala/com/gu/contentapi/porter/graphql/PaginationParameters.scala index 0ba8614..10f9b62 100644 --- a/src/main/scala/com/gu/contentapi/porter/graphql/PaginationParameters.scala +++ b/src/main/scala/com/gu/contentapi/porter/graphql/PaginationParameters.scala @@ -17,8 +17,9 @@ object PaginationParameters { object OrderDateSchema { val definition = EnumType( "OrderDate", - Some("Which date field to use for ordering the content"), + Some("Which date field to use for ordering the content, or whether to search on document score"), List( + EnumValue("score", Some("Ignore when the content was made or published and sort by relevance to the query parameters"), "score"), EnumValue("published", Some("When the content was published to web"), "webPublicationDate"), EnumValue("firstPublished", Some("When the first version of this content was published"), "fields.firstPublicationDate"), EnumValue("lastModified", Some("The last time the content was modified prior to publication"), "fields.lastModified"), diff --git a/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala b/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala index 9e03b8c..93ae4f3 100644 --- a/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala +++ b/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala @@ -11,6 +11,8 @@ import scala.concurrent.ExecutionContext.Implicits.global import io.circe.generic.auto._ import org.slf4j.LoggerFactory +import scala.concurrent.Future + object RootQuery { private val logger = LoggerFactory.getLogger(getClass) @@ -36,20 +38,29 @@ object RootQuery { Field("matchingAnyTag", ArticleEdge, Some("Content which matches any of the tags returned"), arguments= ContentQueryParameters.AllContentQueryParameters, resolve = { ctx=> - ctx.ctx.repo.marshalledDocs(ctx arg ContentQueryParameters.QueryString, - queryFields=ctx arg ContentQueryParameters.QueryFields, - atomId = None, - forChannel = ctx arg ContentQueryParameters.ChannelArg, - userTier = ctx.ctx.userTier, - tagIds = Some(ctx.value.nodes.map(_.id)), - excludeTags = ctx arg ContentQueryParameters.ExcludeTagArg, - sectionIds = ctx arg ContentQueryParameters.SectionArg, - excludeSections = ctx arg ContentQueryParameters.ExcludeSectionArg, - orderDate = ctx arg PaginationParameters.OrderDate, - orderBy = ctx arg PaginationParameters.OrderBy, - limit = ctx arg PaginationParameters.Limit, - cursor = ctx arg PaginationParameters.Cursor, - ) + if(ctx.value.nodes.isEmpty) { + Future(Edge[Content]( + 0L, + None, + false, + Seq() + )) + } else { + ctx.ctx.repo.marshalledDocs(ctx arg ContentQueryParameters.QueryString, + queryFields = ctx arg ContentQueryParameters.QueryFields, + atomId = None, + forChannel = ctx arg ContentQueryParameters.ChannelArg, + userTier = ctx.ctx.userTier, + tagIds = Some(ctx.value.nodes.map(_.id)), + excludeTags = ctx arg ContentQueryParameters.ExcludeTagArg, + sectionIds = ctx arg ContentQueryParameters.SectionArg, + excludeSections = ctx arg ContentQueryParameters.ExcludeSectionArg, + orderDate = ctx arg PaginationParameters.OrderDate, + orderBy = ctx arg PaginationParameters.OrderBy, + limit = ctx arg PaginationParameters.Limit, + cursor = ctx arg PaginationParameters.Cursor, + ) + } }) ) ) diff --git a/src/main/scala/datastore/ElasticsearchRepo.scala b/src/main/scala/datastore/ElasticsearchRepo.scala index ed5d00e..c7bd8fb 100644 --- a/src/main/scala/datastore/ElasticsearchRepo.scala +++ b/src/main/scala/datastore/ElasticsearchRepo.scala @@ -73,8 +73,9 @@ class ElasticsearchRepo(endpoint:ElasticNodeEndpoint, val defaultPageSize:Int=20 private def defaultingSortParam(orderDate:Option[String], orderBy:Option[SortOrder]): Sort = { orderDate match { + case Some("score")=>ScoreSort(orderBy.getOrElse(SortOrder.DESC)) case Some(field)=>FieldSort(field, order = orderBy.getOrElse(SortOrder.DESC)) - case None=>ScoreSort(orderBy.getOrElse(SortOrder.DESC)) + case None=>FieldSort("webPublicationDate", order=orderBy.getOrElse(SortOrder.DESC)) } } From 96d0bbb41827eb8c011995d345689134b1245fb6 Mon Sep 17 00:00:00 2001 From: Andy Gallagher Date: Wed, 17 Jul 2024 15:54:47 +0100 Subject: [PATCH 6/6] Improvements to the inline doc --- .../contentapi/porter/graphql/ContentQueryParameters.scala | 2 +- .../scala/com/gu/contentapi/porter/graphql/RootQuery.scala | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/scala/com/gu/contentapi/porter/graphql/ContentQueryParameters.scala b/src/main/scala/com/gu/contentapi/porter/graphql/ContentQueryParameters.scala index 45d8d82..bd0d5d9 100644 --- a/src/main/scala/com/gu/contentapi/porter/graphql/ContentQueryParameters.scala +++ b/src/main/scala/com/gu/contentapi/porter/graphql/ContentQueryParameters.scala @@ -11,7 +11,7 @@ object ContentQueryParameters { val ContentIdArg = Argument("id", OptionInputType(StringType), description = "get one article by ID") val QueryString = Argument("q", OptionInputType(StringType), description = "an Elastic Search query string to search for content") val QueryFields = Argument("queryFields", OptionInputType(ListInputType(StringType)), description = "fields to perform a query against. Defaults to webTitle and path.") - val TagArg = Argument("tags", OptionInputType(ListInputType(StringType)), description = "look up articles associated with all of these tag IDs") + val TagArg = Argument("tags", OptionInputType(ListInputType(StringType)), description = "look up articles associated with all of these tag IDs. If you don't have exact tag IDs you should instead make a root query on Tags and use the `matchingContent` or `matchingAnyTag` selectors") val ExcludeTagArg = Argument("excludeTags", OptionInputType(ListInputType(StringType)), description = "don't include any articles with these tag IDs") val SectionArg = Argument("sectionId", OptionInputType(ListInputType(StringType)), description = "look up articles in any of these sections") val ExcludeSectionArg = Argument("excludeSections", OptionInputType(ListInputType(StringType)), description = "don't include any articles with these tag IDs") diff --git a/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala b/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala index 93ae4f3..8f41016 100644 --- a/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala +++ b/src/main/scala/com/gu/contentapi/porter/graphql/RootQuery.scala @@ -86,6 +86,8 @@ object RootQuery { val Query = ObjectType[GQLQueryContext, Unit]( "Query", fields[GQLQueryContext, Unit]( Field("article", ArticleEdge, + description = Some("An Article is the main unit of our publication. You can search articles directly here, or query" + + " tags or sections to see what articles live within it."), arguments = ContentQueryParameters.AllContentQueryParameters, resolve = ctx => ctx arg ContentQueryParameters.ContentIdArg match { @@ -101,6 +103,8 @@ object RootQuery { } ), Field("tag", TagEdge, + description = Some("The Guardian uses tags to group similar pieces of content together across multiple different viewpoints. " + + "Tags are a closed set, which can be searched here, and there are different types of tags which represent different viewpoints"), arguments = TagQueryParameters.AllTagQueryParameters, resolve = ctx => ctx.ctx.repo.marshalledTags(ctx arg TagQueryParameters.QueryString, @@ -114,6 +118,8 @@ object RootQuery { ctx arg PaginationParameters.Limit, ctx arg PaginationParameters.Cursor) ), Field("atom", AtomEdge, + description = Some("An Atom is a piece of content which can be linked to multiple articles but may have a production lifecycle independent" + + " of these articles. Examples are cartoons, videos, quizzes, call-to-action blocks, etc."), arguments = AtomQueryParameters.AllParameters, resolve = ctx=> ctx.ctx.repo.atoms(ctx arg AtomQueryParameters.AtomIds, ctx arg AtomQueryParameters.QueryString, ctx arg AtomQueryParameters.QueryFields,