From 5dae01615f214baa0a81401db0816979e7d92baa Mon Sep 17 00:00:00 2001 From: Henry Story Date: Tue, 7 Mar 2023 10:08:02 +0100 Subject: [PATCH 01/42] added ldes example - will need to be re-aranged --- Architecture.md | 4 +- build.sbt | 22 +++ ldes/README.md | 3 + .../src/main/scala/run/cosy/ld/PNGraph.scala | 157 +++++++++++++++++ .../test/scala/run/cosy/ld/FoafWebTest.scala | 87 ++++++++++ .../test/scala/run/cosy/ld/JenaWebTest.scala | 7 + .../test/scala/run/cosy/ld/LdesWebTest.scala | 16 ++ .../test/scala/run/cosy/ld/MiniFoafWWW.scala | 69 ++++++++ .../scala/run/cosy/ld/ldes/MiniLdesWWW.scala | 163 ++++++++++++++++++ .../scala/run/cosy/ld/ldes/prefix/LDES.scala | 23 +++ .../scala/run/cosy/ld/ldes/prefix/SOSA.scala | 46 +++++ .../scala/run/cosy/ld/ldes/prefix/TREE.scala | 42 +++++ .../scala/run/cosy/ld/ldes/prefix/WGS84.scala | 17 ++ project/Dependencies.scala | 16 +- project/build.properties | 2 +- .../main/scala/net/bblfish/app/Wallet.scala | 2 +- .../main/scala/net/bblfish/wallet/AuthN.scala | 27 +++ .../net/bblfish/wallet/BasicAuthWallet.scala | 52 +++--- 18 files changed, 723 insertions(+), 32 deletions(-) create mode 100644 ldes/README.md create mode 100644 ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala create mode 100644 ldes/shared/src/test/scala/run/cosy/ld/FoafWebTest.scala create mode 100644 ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala create mode 100644 ldes/shared/src/test/scala/run/cosy/ld/LdesWebTest.scala create mode 100644 ldes/shared/src/test/scala/run/cosy/ld/MiniFoafWWW.scala create mode 100644 ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala create mode 100644 ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/LDES.scala create mode 100644 ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/SOSA.scala create mode 100644 ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/TREE.scala create mode 100644 ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/WGS84.scala create mode 100644 wallet/shared/src/main/scala/net/bblfish/wallet/AuthN.scala diff --git a/Architecture.md b/Architecture.md index b6503be..a2a224c 100644 --- a/Architecture.md +++ b/Architecture.md @@ -18,7 +18,7 @@ The server can signal its support for any of these via a [WWW-Authenticate](http 3. [HttpSig](https://github.com/bblfish/authentication-panel/blob/main/proposals/HttpSignature.md): the most efficient and secure authentication, best for connecting to other servers 4. UMA: as described in [Solid-OIDC](https://solidproject.org/TR/oidc-primer) is useful as a way of tying into widely deployed OAuth systems, but not efficient for highly decentralised apps. 5. Credential based - find the spec -6. Cookies: once a user is authenticated for a realm, using a cookie may be enough to continue the interaction. +6. Cookies: once a user is authenticated for a realm, using a cookie may be enough to continue the interaction. But cookie use in the browser by non-origin apps is limited and tricky.([see detailed course](https://www.youtube.com/watch?v=34wC1C61lg0)) We will get going with 1 and 3, but keeping 2 and 4 in mind will be helpful to make sure the architecture is correct. In any case the system should be extensible so that others can contribute other auth schemes without problem. @@ -66,7 +66,7 @@ trait Client[F[_]]: //... ``` -This parallels the way Http4s defines server applications. As Ross Baker quipped in his [introductory talk]() ([video](https://www.youtube.com/watch?v=urdtmx4h5LE)): +This parallels the way Http4s defines server applications. As Ross Baker quipped in his introductory talk ([video](https://www.youtube.com/watch?v=urdtmx4h5LE)): > HTTP applications are just a Kleisli function from a streaming request to a polymorphic effect of a streaming response. So what's the problem? diff --git a/build.sbt b/build.sbt index 8bee65b..bcc0dbc 100644 --- a/build.sbt +++ b/build.sbt @@ -101,6 +101,28 @@ lazy val free = crossProject(JVMPlatform) // , JSPlatform) ) ) +// an LDES client +lazy val ldes = crossProject(JVMPlatform) + .crossType(CrossType.Full) + .in(file("ldes")) + .settings(commonSettings: _*) + .settings( + name := "LDES Client", + description := "Linked Data Event Stream Libraries and Client", + // scalacOptions := scala3jsOptions, + resolvers += sonatypeSNAPSHOT, + libraryDependencies ++= Seq( + banana.bananaRdf.value, +// cats.effect.value, + cats.fs2.value + ), + libraryDependencies ++= Seq( + munit.value % Test, + cats.munitEffect.value % Test, + banana.bananaJena.value % Test + ) + ) + //todo: we should split the wallet into client-wallet and the full wallet library // as clients of the wallet only need a minimal interface lazy val wallet = crossProject(JVMPlatform) // , JSPlatform) diff --git a/ldes/README.md b/ldes/README.md new file mode 100644 index 0000000..30588e1 --- /dev/null +++ b/ldes/README.md @@ -0,0 +1,3 @@ +# Linked Data Event Stream + +This is a space to explore a generic LDES library that can explore LDES data, collect it and work with it. This should be RDF library agnostic, language agnostic and HTTP framework-agnostic. That is, all of those should be something one can add later. Furthermore, access control logic should be dealt with transparently at a lower layer. diff --git a/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala b/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala new file mode 100644 index 0000000..0088daf --- /dev/null +++ b/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala @@ -0,0 +1,157 @@ +package run.cosy.ld + +import cats.MonadError +import cats.effect.kernel.Concurrent +import fs2.Pure +import org.w3.banana.RDF.{Graph, Node, Statement, URI} +import org.w3.banana.{Ops, RDF} + +trait Web[F[_]: Concurrent, R <: RDF]: + /** get a Graph for the given URL (without a fragment) */ + def get(url: RDF.URI[R]): F[RDF.Graph[R]] + /** get Pointed Named Graph for given url */ + def getPNG(url: RDF.URI[R]): F[UriNGraph[R]] + + +/** A Pointed Named Graph, ie, a pointer into a NamedGraph We don't use a case class here as + * equality between PNGgraphs is complicated by the need to prove isomorphism between graphs, and + * the nodes have to be equivalent. + * + * todo: it would make sense to have subtypes for different types of points, so + * that filters on points that are urls can be typed. Also when the point is on the + * literal then / makes less sense. png.jump makes sense only if the node is a URL + */ +trait PNGraph[R <: RDF]: + type Point <: Node[R] + def point: Point + def name: URI[R] + def graph: Graph[R] + +trait SubjPNGraph[R <: RDF] extends PNGraph[R]: + type Point <: RDF.Statement.Subject[R] + + def /(rel: RDF.URI[R])(using ops: Ops[R]): Seq[PNGraph[R]] = + import ops.{*, given} + graph.find(point, rel, `*`) + .map { (tr: RDF.Triple[R]) => + tr.obj.asNode.fold( + uri => new UriNGraph(uri,name,graph), + bn => new BNodeNGraph[R](bn, name, graph), + lit => new LiteralNGraph[R](lit, name, graph) + ) + } + .toSeq + end / + + inline def rel(rel: RDF.URI[R])(using ops: Ops[R]): Seq[PNGraph[R]] = /(rel) + + def \(rel: RDF.URI[R])(using ops: Ops[R]): Seq[PNGraph[R]] = + import ops.{*, given} + graph.find(`*`, rel, point) + .map { (tr: RDF.Triple[R]) => + val s: Statement.Subject[R] = tr.subj + s.foldSubj[PNGraph[R]]( + uri => new UriNGraph(uri, name, graph), + bn => new BNodeNGraph[R](bn, name, graph) + ) + } + .toSeq + end \ + +end SubjPNGraph + + +/** A PNG where the point is a URI */ +class UriNGraph[R <: RDF]( + val point: URI[R], + val name: URI[R], + val graph: Graph[R] +)(using ops: Ops[R]) extends SubjPNGraph[R]: + import ops.{*,given} + type Point = RDF.URI[R] + + lazy val pointLoc: RDF.URI[R] = point.fragmentLess + lazy val isLocal: Boolean = pointLoc == name + +//todo: things are more complex when one takes redirects into account... + def jump[F[_]: Concurrent](using web: Web[F, R]): F[UriNGraph[R]] = + import cats.syntax.functor.* + import ops.{*, given} + if isLocal then summon[Concurrent[F]].pure(this) + else web.get(pointLoc).map(g => new UriNGraph(point, pointLoc, g)) + +end UriNGraph + + +/** A PNG where the point is a BNode */ +class BNodeNGraph[R <: RDF]( + val point: RDF.BNode[R], + val name: URI[R], + val graph: Graph[R] +) extends SubjPNGraph[R]: + type Point = RDF.BNode[R] + + +/** A PNG where the point is a BNode */ +class LiteralNGraph[R <: RDF]( + val point: RDF.Literal[R], + val name: URI[R], + val graph: Graph[R] +) extends PNGraph[R]: + type Point = RDF.Literal[R] + + //todo: this is duplicated code with SubjPNGraph. + def \(rel: RDF.URI[R])(using ops: Ops[R]): Seq[PNGraph[R]] = + import ops.{*, given} + graph.find(`*`, rel, point) + .map { (tr: RDF.Triple[R]) => + tr.subj.foldSubj( + uri => UriNGraph(uri, name, graph), + bn => BNodeNGraph(bn, name, graph) + ) + }.toSeq +end LiteralNGraph + + +object PNGraph: + + extension [F[_]: cats.effect.Concurrent, R <: RDF]( + pngs: Seq[PNGraph[R]] + )(using web: Web[F, R], ops: Ops[R]) + + def jump : fs2.Stream[F, PNGraph[R]] = + import ops.given + var local: List[PNGraph[R]] = Nil + var remoteGs: List[UriNGraph[R]] = Nil + pngs.foreach { + case ug: UriNGraph[R] if !ug.isLocal => remoteGs = ug :: remoteGs + case other => local = other::local + } + import cats.syntax.all.given + + val in: fs2.Stream[Pure, PNGraph[R]] = fs2.Stream.chunk(fs2.Chunk.seq(local)) + val out: fs2.Stream[F, PNGraph[R]] = + fs2.Stream + .emits(remoteGs) + // todo: group the urls by domain (groupAdjacentBy) and run each with two threads max + .parEvalMapUnordered(5) { (remote: UriNGraph[R]) => + // todo: we should deal with errors fetching graphs + // web should perhaps return a UriNGraph + web.get(remote.pointLoc).map(g => UriNGraph[R](remote.point, remote.pointLoc, g)) + } + in ++ out + + end jump + + + extension[F[_] : cats.effect.Concurrent, R <: RDF] ( + pngs: fs2.Stream[F, PNGraph[R]] + )(using ops: Ops[R]) + + def /(rel: RDF.URI[R]): fs2.Stream[F, PNGraph[R]] = + pngs.collect{ + case sub: SubjPNGraph[R] => fs2.Stream(sub/rel*) + }.flatten + + inline def rel(rel: RDF.URI[R]): fs2.Stream[F, PNGraph[R]] = /(rel) + diff --git a/ldes/shared/src/test/scala/run/cosy/ld/FoafWebTest.scala b/ldes/shared/src/test/scala/run/cosy/ld/FoafWebTest.scala new file mode 100644 index 0000000..7a6acce --- /dev/null +++ b/ldes/shared/src/test/scala/run/cosy/ld/FoafWebTest.scala @@ -0,0 +1,87 @@ +package run.cosy.ld + +import cats.effect.IO +import cats.effect.kernel.Concurrent +import org.w3.banana.RDF +import org.w3.banana.Ops +import munit.CatsEffectSuite +import run.cosy.ld.MiniFoafWWW.EricP + +trait FoafWebTest[R <: RDF]()(using ops: Ops[R]) extends CatsEffectSuite { + val miniWeb = new MiniFoafWWW[R] + import miniWeb.foaf + given www: Web[IO,R] = miniWeb + import ops.{*, given} + import PNGraph.{given,*} + import MiniFoafWWW.* + import cats.effect.IO.asyncForIO + + test("find bbl friends WebIDs inside graph (no jump)") { + + val bblng: IO[UriNGraph[R]] = www.getPNG(URI(Bbl)) + bblng.map { bblNg => + //todo: make it possible to just get nodes + val kns: Seq[String] = (bblNg/foaf.knows).collect{ + case u: UriNGraph[R] => u.point.value + } + assertEquals(Set(kns*), Set(EricP,Timbl,CSarven)) + } + + } + + test("find bblFriend WebIDs after jumping") { + www.getPNG(URI(Bbl)).flatMap { bblNg => + val friendsDef: fs2.Stream[IO, PNGraph[R]] = (bblNg / foaf.knows).jump + val expectedURIs: Set[String] = Set(EricP, Timbl, CSarven) + val result1: IO[Set[String]] = friendsDef.compile.toList.map(pnglst => + val uris: Seq[String] = pnglst.collect { case u: UriNGraph[R] => u.point.value } + Set(uris *) + ) + result1.map(set => assertEquals(set, expectedURIs)) + } + + //todo: test what happens if a WebID is broken. + } + + test("find canonical names of friends (as defined in their profile)") { + www.getPNG(URI(Bbl)).flatMap { bblNg => + //most of the names are only available from the definitional graphs + val names: fs2.Stream[IO, String] = (bblNg / foaf.knows).jump.collect { + case ug: UriNGraph[R] => fs2.Stream(ug / foaf.name *) + }.flatten.collect { case litG: LiteralNGraph[R] => litG.point.text } + names.compile.toList.map(lst => + assertEquals(Set(lst*), Set( + "Tim Berners-Lee", "Eric Prud'hommeaux", "Sarven Capadisli"))) + } + } + + + test("find all names of friends (local and remote)") { + www.getPNG(URI(Bbl)).flatMap { bblNg => + //most of the names are only available from the definitional graphs + val allNames: fs2.Stream[IO, String] = (bblNg / foaf.knows).jump.collect { + case ug: SubjPNGraph[R] => fs2.Stream(ug / foaf.name *) + }.flatten.collect { case litG: LiteralNGraph[R] => litG.point.text } + + allNames.compile.toList.map { lst => + assertEquals(Set(lst*), Set( + "Tim Berners-Lee", "Eric Prud'hommeaux", "Sarven Capadisli", "James Gosling")) + } + } + } + + test("find all names of friends (local and remote) - shorter version") { + www.getPNG(URI(Bbl)).flatMap { bblNg => + //most of the names are only available from the definitional graphs + val allNames: fs2.Stream[IO, String] = + bblNg.rel(foaf.knows).jump + .rel(foaf.name) + .collect { case litG: LiteralNGraph[R] => litG.point.text } + + allNames.compile.toList.map { lst => + assertEquals(Set(lst *), Set( + "Tim Berners-Lee", "Eric Prud'hommeaux", "Sarven Capadisli", "James Gosling")) + } + } + } +} diff --git a/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala b/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala new file mode 100644 index 0000000..bee9bea --- /dev/null +++ b/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala @@ -0,0 +1,7 @@ +package run.cosy.ld + +import org.w3.banana.jena.JenaRdf.{R, given} + + +class JenaFoafWebTest extends FoafWebTest[R]() + diff --git a/ldes/shared/src/test/scala/run/cosy/ld/LdesWebTest.scala b/ldes/shared/src/test/scala/run/cosy/ld/LdesWebTest.scala new file mode 100644 index 0000000..420bf2a --- /dev/null +++ b/ldes/shared/src/test/scala/run/cosy/ld/LdesWebTest.scala @@ -0,0 +1,16 @@ +package run.cosy.ld +import cats.effect.IO +import org.w3.banana.RDF +import org.w3.banana.Ops +import run.cosy.ld.ldes.MiniLdesWWW +import munit.CatsEffectSuite + +trait LdesWebTest[R <: RDF]()(using ops: Ops[R]) extends CatsEffectSuite: + + val miniWeb = new MiniLdesWWW[R] + import miniWeb.foaf + given www: Web[IO, R] = miniWeb + + test("") { + + } diff --git a/ldes/shared/src/test/scala/run/cosy/ld/MiniFoafWWW.scala b/ldes/shared/src/test/scala/run/cosy/ld/MiniFoafWWW.scala new file mode 100644 index 0000000..5fe4472 --- /dev/null +++ b/ldes/shared/src/test/scala/run/cosy/ld/MiniFoafWWW.scala @@ -0,0 +1,69 @@ +package run.cosy.ld + +import cats.Id +import cats.effect.IO +import io.lemonlabs.uri.AbsoluteUrl +import org.w3.banana.{Ops, RDF} +import org.w3.banana.prefix +import org.w3.banana.diesel +import org.w3.banana.diesel.{*, given} + +object MiniFoafWWW: + val BblCard = "https://bblfish.net/people/henry/card" + val Bbl = BblCard + "#me" + val TimblCard = "https://www.w3.org/People/Berners-Lee/card" + val Timbl = TimblCard + "#i" + val EricPCard = "https://www.w3.org/People/Eric/ericP-foaf.rdf" + val EricP = EricPCard + "#ericP" + val CSarvenCard = "https://csarven.ca/" + val CSarven = CSarvenCard + "#i" + +class MiniFoafWWW[R <: RDF](using ops: Ops[R]) extends Web[IO, R]: + import ops.{*, given} + import MiniFoafWWW.* + + val foaf = prefix.FOAF[R] + val fr = Lang("fr") // todo, put together a list of Lang constants + val en = Lang("en") + + def getPNG(url: RDF.URI[R]): IO[UriNGraph[R]] = + val doc = url.fragmentLess + get(doc).map(g => new UriNGraph(url, doc, g)) + + def get(url: RDF.URI[R]): IO[RDF.Graph[R]] = + import scala.language.implicitConversions + val res: RDF.rGraph[R] = + url.value match + case BblCard => + (rURI("#me") -- foaf.name ->- "Henry Story".lang(en) + -- foaf.knows ->- URI(EricP) + -- foaf.knows ->- URI(CSarven) + -- foaf.knows ->- URI(Timbl) + -- foaf.knows ->- ( BNode() -- foaf.name ->- "James Gosling") + ).graph + case TimblCard => + (rURI("#i") -- foaf.name ->- "Tim Berners-Lee".lang(en) + -- foaf.knows ->- URI(Bbl) + -- foaf.knows ->- URI(CSarven) + -- foaf.knows ->- URI(EricP) + -- foaf.knows ->- ( BNode() -- foaf.name ->- "Vint Cerf") + ).graph + case EricPCard => + (rURI("#ericP") -- foaf.name ->- "Eric Prud'hommeaux".lang(fr) + -- foaf.knows ->- URI(Bbl) + -- foaf.knows ->- URI(CSarven) + -- foaf.knows ->- URI(Timbl) + ).graph + case CSarvenCard => + (rURI("#i") -- foaf.name ->- "Sarven Capadisli".lang(en) + -- foaf.knows ->- URI(Bbl) + -- foaf.knows ->- URI(Timbl) + -- foaf.knows ->- URI(EricP) + ).graph + + + case _ => ops.rGraph.empty + + IO(res.resolveAgainst(AbsoluteUrl.parse(url.value))) + end get + \ No newline at end of file diff --git a/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala b/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala new file mode 100644 index 0000000..f2b6d45 --- /dev/null +++ b/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala @@ -0,0 +1,163 @@ +package run.cosy.ld.ldes + +import cats.effect.IO +import io.lemonlabs.uri.AbsoluteUrl +import org.w3.banana.diesel.{*, given} +import org.w3.banana.{diesel, *} +import run.cosy.ld.* +import run.cosy.ld.ldes.prefix as ldesPre + +object MiniLdesWWW: + def mechelen(date: String): String = "https://ldes.mechelen.org/" + date + val D09_05: String = mechelen("2021-09-05") + val D09_06: String = mechelen("2021-09-06") + val D09_07: String = mechelen("2021-09-07") + val Container: String = mechelen("") + +class MiniLdesWWW[R <: RDF](using ops: Ops[R]) extends Web[IO, R]: + import MiniLdesWWW.* + import ops.{*, given} + val foaf = prefix.FOAF[R] + val tree = ldesPre.TREE[R] + val sosa = ldesPre.SOSA[R] + val wgs84 = ldesPre.WGS84[R] + val ldes = ldesPre.LDES[R] + + def area(loc: String) = rURI("area#d" + loc) + def pzDev(num: String) = URI("https://data.politie.be/sensor/dev#" + num) + def polMsr(tp: String) = URI("https://data.politie.be/sensors/measurement#" + tp) + def crop(area: String) = URI("https://data.cropland.be/area#" + area) + def cropProp(prop: String) = URI("https://data.cropland.be/measure#" + prop) + + def getPNG(url: RDF.URI[R]): IO[UriNGraph[R]] = + val doc = url.fragmentLess + get(doc).map(g => new UriNGraph(url, doc, g)) + + def get(url: RDF.URI[R]): IO[RDF.Graph[R]] = + import scala.language.implicitConversions + val res: RDF.rGraph[R] = + url.value match + case Container => + (rURI("").a(ldes.EventStream) + -- ldes.timestampPath ->- sosa.resultTime + -- tree.shape ->- rURI("flows-shacl") + -- tree.view ->- rURI("2021-09-05") + ).graph + case D09_05 => + (rURI("") -- rdf.typ ->- tree.Node + -- tree.relation ->- ( + BNode() -- rdf.typ ->- tree.GreaterThanRelation + -- tree.node ->- rURI("2021-09-06") + -- tree.path ->- sosa.resultTime + -- tree.value ->- ("2021-09-06T00:00:00+02" ^^ xsd.dateTimeStamp) + )).graph ++ ( + rURI("#3") -- rdf.typ ->- sosa.Observation + -- wgs84.location ->- area("loc781089") + -- sosa.hasSimpleResult ->- ("4.0" ^^ xsd.float) + -- sosa.madeBySensor ->- pzDev("213501") + -- sosa.observedProperty ->- polMsr("motorized") + -- sosa.resultTime ->- ("2021-09-05T23:00:00+02" ^^ xsd.dateTimeStamp) + ).graph.triples.toSeq ++ ( + rURI("#482") -- rdf.typ ->- sosa.Observation + -- wgs84.location ->- area("loc") + -- sosa.hasSimpleResult ->- ("2455.1123" ^^ xsd.float) + -- sosa.madeBySensor ->- crop("schoolstraat") + -- sosa.observedProperty ->- cropProp("deviceNbr") + -- sosa.resultTime ->- ("2021-09-05T22:30:00+02" ^^ xsd.dateTimeStamp) + ).graph.triples.toSeq ++ ( + rURI("#4464") -- rdf.typ ->- sosa.Observation + -- wgs84.location ->- area("loc734383") + -- sosa.hasSimpleResult ->- ("10.0" ^^ xsd.float) + -- sosa.madeBySensor ->- pzDev("213504+5+6") + -- sosa.observedProperty ->- polMsr("bike") + -- sosa.resultTime ->- ("2021-09-05T23:00:00+02" ^^ xsd.dateTimeStamp) + ).graph.triples.toSeq ++ ( + rURI(".").a(ldes.EventStream) + -- ldes.timestampPath ->- sosa.resultTime + -- tree.shape ->- rURI("flows-shacl") + -- tree.view ->- rURI("") + -- tree.member ->- rURI("#3") + -- tree.member ->- rURI("#482") + -- tree.member ->- rURI("#4464") + ).graph.triples.toSeq + case D09_06 => + (rURI("").a(tree.Node) + -- tree.relation ->- ( + BNode().a(tree.LessThanRelation) + -- tree.node ->- rURI("2021-09-05") + -- tree.path ->- sosa.resultTime + -- tree.value ->- ("2021-09-06T00:00:00+02" ^^ xsd.dateTimeStamp) + ) + -- tree.relation ->- ( + BNode().a(tree.GreaterThanRelation) + -- tree.node ->- rURI("2021-09-07") + -- tree.path ->- sosa.resultTime + -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) + )).graph ++ ( + rURI("#3003").a(sosa.Observation) + -- wgs84.location ->- area("loc763628") + -- sosa.hasSimpleResult ->- ("44.0" ^^ xsd.float) + -- sosa.madeBySensor ->- pzDev("213503") + -- sosa.observedProperty ->- polMsr("motorized") + -- sosa.resultTime ->- ("2021-09-06T11:00:00+02" ^^ xsd.dateTimeStamp) + ).graph.triples.toSeq ++ ( + rURI("#4493").a(sosa.Observation) + -- wgs84.location ->- area("loc734383") + -- sosa.hasSimpleResult ->- ("197.0" ^^ xsd.float) + -- sosa.madeBySensor ->- pzDev("213504+5+6") + -- sosa.observedProperty ->- polMsr("motorized") + -- sosa.resultTime ->- ("2021-09-06T12:00:00+02" ^^ xsd.dateTimeStamp) + ).graph.triples.toSeq ++ ( + rURI("#48").a(sosa.Observation) + -- wgs84.location ->- area("loc781089") + -- sosa.hasSimpleResult ->- ("1.0" ^^ xsd.float) + -- sosa.madeBySensor ->- pzDev("213501") + -- sosa.observedProperty ->- polMsr("bike") + -- sosa.resultTime ->- ("2021-09-06T22:00:00+02" ^^ xsd.dateTimeStamp) + ).graph.triples.toSeq ++ ( + rURI(".").a(ldes.EventStream) + -- ldes.timestampPath ->- sosa.resultTime + -- tree.shape ->- rURI("flows-shacl") + -- tree.view ->- rURI("") + -- tree.member ->- rURI("#4493") + -- tree.member ->- rURI("#48") + -- tree.member ->- rURI("#3003") + ).graph.triples.toSeq + case D09_07 => + (rURI("").a(tree.Node) + -- tree.relation ->- ( + BNode().a(tree.LessThanRelation) + -- tree.node ->- rURI("2021-09-07") + -- tree.path ->- sosa.resultTime + -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) + )).graph ++ ( + rURI("#658").a(sosa.Observation) + -- wgs84.location ->- area("loc") + -- sosa.hasSimpleResult ->- ("5087.4795" ^^ xsd.float) + -- sosa.madeBySensor ->- crop("schoolstraat") + -- sosa.observedProperty ->- cropProp("deviceNbr") + -- sosa.resultTime ->- ("2021-09-07T18:30:00+02" ^^ xsd.dateTimeStamp) + ).graph.triples.toSeq ++ ( + rURI("#637").a(sosa.Observation) + -- wgs84.location ->- area("loc") + -- sosa.hasSimpleResult ->- ("7009.3345" ^^ xsd.float) + -- sosa.madeBySensor ->- crop("schoolstraat") + -- sosa.observedProperty ->- cropProp("deviceNbr") + -- sosa.resultTime ->- ("2021-09-07T13:15:00+02" ^^ xsd.dateTimeStamp) + ).graph.triples.toSeq ++ ( + rURI("#3074").a(sosa.Observation) + -- wgs84.location ->- area("loc763628") + -- sosa.hasSimpleResult ->- ("1.0" ^^ xsd.float) + -- sosa.madeBySensor ->- pzDev("213503") + -- sosa.observedProperty ->- polMsr("bike") + -- sosa.resultTime ->- ("2021-09-06T22:00:00+02" ^^ xsd.dateTimeStamp) + ).graph.triples.toSeq ++ ( + rURI(".").a(ldes.EventStream) + -- ldes.timestampPath ->- sosa.resultTime + -- tree.shape ->- rURI("flows-shacl") + -- tree.view ->- rURI("") + -- tree.member ->- rURI("#658") + -- tree.member ->- rURI("#3074") + -- tree.member ->- rURI("#637") + ).graph.triples.toSeq + IO(res.resolveAgainst(AbsoluteUrl.parse(url.value))) diff --git a/ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/LDES.scala b/ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/LDES.scala new file mode 100644 index 0000000..342842e --- /dev/null +++ b/ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/LDES.scala @@ -0,0 +1,23 @@ +package run.cosy.ld.ldes.prefix + +import org.w3.banana.{Ops, RDF, PrefixBuilder} + +object LDES: + def apply[Rdf <: RDF](using ops: Ops[Rdf]) = new LDES() + +class LDES[Rdf <: RDF](using ops: Ops[Rdf]) + extends PrefixBuilder[Rdf]("ldes", ops.URI("https://w3id.org/ldes#")): + + val EventStream = apply("EventStream") + val EventSource = apply("EventSource") + val RetentionPolicy = apply("RetentionPolicy") + val LatestVersionSubset = apply("LatestVersionSubset") + val DurationAgoPolicy = apply("DurationAgoPolicy") + val retentionPolicy = apply("retentionPolicy") + val amount = apply("amount") + val versionKey = apply("versionKey") + val versionOfPath = apply("versionOfPath") + val timestampPath = apply("timestampPath") + val versionMaterializationOf = apply("versionMaterializationOf") + val versionMaterializationUntil = apply("versionMaterializationUntil") + diff --git a/ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/SOSA.scala b/ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/SOSA.scala new file mode 100644 index 0000000..aee894f --- /dev/null +++ b/ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/SOSA.scala @@ -0,0 +1,46 @@ +package run.cosy.ld.ldes.prefix + +import org.w3.banana.{Ops, PrefixBuilder, RDF} + +object SOSA: + def apply[Rdf <: RDF](using ops: Ops[Rdf]) = new SOSA() + +class SOSA[Rdf <: RDF](using ops: Ops[Rdf]) + extends PrefixBuilder[Rdf]("tree", ops.URI("http://www.w3.org/ns/sosa/")): + + val FeatureOfInterest = apply("FeatureOfInterest") + val ObservableProperty = apply("ObservableProperty") + val ActuatableProperty = apply("ActuatableProperty") + val Sample = apply("Sample") + val hasSample = apply("hasSample") + val isSampleOf = apply("isSampleOf") + val Platform = apply("Platform") + val hosts = apply("hosts") + val isHostedBy = apply("isHostedBy") + val Procedure = apply("Procedure") + val Sensor = apply("Sensor") + val observes = apply("observes") + val isObservedBy = apply("isObservedBy") + val Actuator = apply("Actuator") + val Sampler = apply("Sampler") + val usedProcedure = apply("usedProcedure") + val hasFeatureOfInterest = apply("hasFeatureOfInterest") + val isFeatureOfInterestOf = apply("isFeatureOfInterestOf") + val Observation = apply("Observation") + val madeObservation = apply("madeObservation") + val madeBySensor = apply("madeBySensor") + val observedProperty = apply("observedProperty") + val Actuation = apply("Actuation") + val madeActuation = apply("madeActuation") + val madeByActuator = apply("madeByActuator") + val actsOnProperty = apply("actsOnProperty") + val isActedOnBy = apply("isActedOnBy") + val Sampling = apply("Sampling") + val madeSampling = apply("madeSampling") + val madeBySampler = apply("madeBySampler") + val Result = apply("Result") + val hasResult = apply("hasResult") + val isResultOf = apply("isResultOf") + val hasSimpleResult = apply("hasSimpleResult") + val resultTime = apply("resultTime") + val phenomenonTime = apply("phenomenonTime") diff --git a/ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/TREE.scala b/ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/TREE.scala new file mode 100644 index 0000000..60cfb85 --- /dev/null +++ b/ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/TREE.scala @@ -0,0 +1,42 @@ +package run.cosy.ld.ldes.prefix + +import org.w3.banana.{Ops, PrefixBuilder, RDF} + +object TREE: + def apply[Rdf <: RDF](using ops: Ops[Rdf]) = new TREE() + +class TREE[Rdf <: RDF](using ops: Ops[Rdf]) + extends PrefixBuilder[Rdf]("tree", ops.URI("https://w3id.org/tree#")): + + val Collection = apply("Collection") + val ViewDescription = apply("ViewDescription") + val Node = apply("Node") + val Relation = apply("Relation") + val ConditionalImport = apply("ConditionalImport") + val PrefixRelation = apply("PrefixRelation") + val SubstringRelation = apply("SubstringRelation") + val SuffixRelation = apply("SuffixRelation") + val GreaterThanRelation = apply("GreaterThanRelation") + val GreaterThanOrEqualToRelation = apply("GreaterThanOrEqualToRelation") + val LessThanRelation = apply("LessThanRelation") + val LessThanOrEqualToRelation = apply("LessThanOrEqualToRelation") + val EqualToRelation = apply("EqualToRelation") + val GeospatiallyContainsRelation = apply("GeospatiallyContainsRelation") + val InBetweenRelation = apply("InBetweenRelation") + val viewDescription = apply("viewDescription") + val relation = apply("relation") + val remainingItems = apply("remainingItems") + val node = apply("node") + val value = apply("value") + val path = apply("path") + val view = apply("view") + val member = apply("member") + val search = apply("search") + val shape = apply("shape") + val conditionalImport = apply("conditionalImport") + val zoom = apply("zoom") + val longitudeTile = apply("longitudeTile") + val latitudeTile = apply("latitudeTile") + val timeQuery = apply("timeQuery") + + \ No newline at end of file diff --git a/ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/WGS84.scala b/ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/WGS84.scala new file mode 100644 index 0000000..2693143 --- /dev/null +++ b/ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/WGS84.scala @@ -0,0 +1,17 @@ +package run.cosy.ld.ldes.prefix + +import org.w3.banana.PrefixBuilder +import org.w3.banana.{RDF,Ops} + +object WGS84: + def apply[R <: RDF](using Ops[R]) = new WGS84() + +class WGS84[R <: RDF](using ops: Ops[R]) + extends PrefixBuilder[R]("wgs84", ops.URI("http://www.w3.org/2003/01/geo/wgs84_pos#")): + + lazy val Point = apply("Point") + lazy val SpatialThing = apply("SpatialThing") + lazy val alt = apply("alt") + lazy val lat = apply("lat") + lazy val lat_long = apply("lat_long") + lazy val location = apply("location") diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 2b72e5b..3c303dd 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -3,11 +3,11 @@ import sbt.{Def, _} object Dependencies { object Ver { - val scala = "3.2.1" - val http4s = "1.0.0-M37" - val banana = "0.9-baf7258-SNAPSHOT" + val scala = "3.2.2" + val http4s = "1.0.0-M39" + val banana = "0.9-79e8845-20230228T213613Z-SNAPSHOT" val bobcats = "0.3-3236e64-SNAPSHOT" - val httpSig = "0.4-b4ee7cc-20221221T174054Z-SNAPSHOT" + val httpSig = "0.4-ac23f8b-SNAPSHOT" } object other { @@ -30,8 +30,10 @@ object Dependencies { } object cats { - lazy val core = Def.setting("org.typelevel" %%% "cats-core" % "2.8.0") - lazy val free = Def.setting("org.typelevel" %%% "cats-free" % "2.8.0") + lazy val core = Def.setting("org.typelevel" %%% "cats-core" % "2.9.0") + lazy val free = Def.setting("org.typelevel" %%% "cats-free" % "2.9.0") +// lazy val effect = Def.setting("org.typelevel" %% "cats-effect" % "3.4.8") + lazy val fs2 = Def.setting("co.fs2" %% "fs2-core" % "3.6.1") // https://github.com/typelevel/munit-cats-effect lazy val munitEffect = @@ -52,6 +54,8 @@ object Dependencies { // not published yet object banana { + lazy val bananaRdf = + Def.setting("net.bblfish.rdf" %%% "banana-rdf" % Ver.banana) lazy val bananaJena = Def.setting("net.bblfish.rdf" %%% "banana-jena-io-sync" % Ver.banana) } diff --git a/project/build.properties b/project/build.properties index 8b9a0b0..46e43a9 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.8.0 +sbt.version=1.8.2 diff --git a/wallet/shared/src/main/scala/net/bblfish/app/Wallet.scala b/wallet/shared/src/main/scala/net/bblfish/app/Wallet.scala index 7941d12..fd418aa 100644 --- a/wallet/shared/src/main/scala/net/bblfish/app/Wallet.scala +++ b/wallet/shared/src/main/scala/net/bblfish/app/Wallet.scala @@ -28,5 +28,5 @@ trait Wallet[F[_]] { * Note: For this to work, I think we need to assume that the URL in the Request is absolute. see * [[https://github.com/http4s/http4s/discussions/5930#discussioncomment-3777066 cats-uri discussion]] */ - def sign(res: Response[F], req: Request[F]): F[Request[F]] + def sign(failed: Response[F], lastReq: Request[F]): F[Request[F]] } diff --git a/wallet/shared/src/main/scala/net/bblfish/wallet/AuthN.scala b/wallet/shared/src/main/scala/net/bblfish/wallet/AuthN.scala new file mode 100644 index 0000000..f57f0f7 --- /dev/null +++ b/wallet/shared/src/main/scala/net/bblfish/wallet/AuthN.scala @@ -0,0 +1,27 @@ +package net.bblfish.wallet + +import cats.Id +import org.http4s.Request +import org.http4s.headers.Authorization + +/** + * The signature for authenticating a Request. + * + * @tparam F The monad for the Stream + * @tparam S ASync monad for time it takes to get signature - could be Id monad on many platforms + */ +trait AuthN[F[_], S[_]]: + /** Signing the request + * This could be a function, but on some platforms and for some signing algorithms + * the signing may need to be asynchronous in S[_]. The internal monad F on the other + * had will be asyncrhonous, as it is needed for streaming responses. + */ + def sign(originalReq: Request[F]): S[Request[F]] + + +class Basic[F[_]](username: String, pass: String) extends AuthN[F, Id]: + override + def sign(originalReq: Request[F]): Id[Request[F]] = + originalReq.withHeaders( + Authorization(org.http4s.BasicCredentials(username, pass)) + ) diff --git a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala index 058b952..6f36ad0 100644 --- a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala +++ b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala @@ -91,22 +91,30 @@ trait ChallengeResponse: response: h4s.Response[F] ): F[h4s.Request[F]] -/* - * First attempt at a Wallet, just to get things going. - * The wallet must be given callections of passwords for domains, KeyIDs for - * http signatures - * it needs a client to follow links to the WAC rules, though it would be better - * if instead it were given a DataSet proxied to the web to allow caching. - */ +/** + * First attempt at a Wallet, just to get things going. + * The wallet must be given collections of passwords for domains, KeyIDs for + * http signatures, cookies (!), OpenId info, ... + * + * Note that CookieJar is a algebra for a mutable used to produce a middleware. + * So perhaps a wallet takes a set of middlewares + * each adapted for a particular situation, and it would proceed as follows: + * + * + * it needs a client to follow links to the WAC rules, though it may be better + * if instead it were given a DataSet proxied to the web to allow caching. + * On the other hand with free monads one could have those be interpreted according + * to context... Todo: compare this way of working with free-monads. + */ class BasicWallet[F[_], Rdf <: RDF]( - db: Map[ll.Authority, BasicId], - keyIdDB: Seq[KeyData[F]] -)(using - client: Client[F], - ops: Ops[Rdf], - rdfDecoders: RDFDecoders[F, Rdf], - fc: Concurrent[F], - clock: Clock[F] + db: Map[ll.Authority, BasicId], + keyIdDB: Seq[KeyData[F]] +)(client: Client[F])( + using + ops: Ops[Rdf], + rdfDecoders: RDFDecoders[F, Rdf], + fc: Concurrent[F], + clock: Clock[F] ) extends Wallet[F]: val reqSel: ReqSelectors[H4] = new ReqSelectors[H4](using new SelectorFnsH4()) @@ -241,26 +249,26 @@ class BasicWallet[F[_], Rdf <: RDF]( * quad store.) */ override def sign( - res: h4s.Response[F], - req: h4s.Request[F] + failed: h4s.Response[F], + lastReq: h4s.Request[F] ): F[h4s.Request[F]] = import cats.syntax.applicativeError.given - res.status.code match + failed.status.code match case 402 => fc.raiseError( new Exception("We don't support payment authentication yet") ) case 401 => - res.headers.get[h4hdr.`WWW-Authenticate`] match + failed.headers.get[h4hdr.`WWW-Authenticate`] match case None => fc.raiseError( new Exception("No WWW-Authenticate header. Don't know how to login") ) case Some(h4hdr.`WWW-Authenticate`(nel)) => // do we recognise a method? for - url <- fc.fromTry(Try(http4sUrlToLLUrl(req.uri).toAbsoluteUrl)) - authdReq <- fc.fromTry(basicChallenge(url.authority, req, nel)).handleErrorWith { _ => - httpSigChallenge(url, req, res, nel) + url <- fc.fromTry(Try(http4sUrlToLLUrl(lastReq.uri).toAbsoluteUrl)) + authdReq <- fc.fromTry(basicChallenge(url.authority, lastReq, nel)).handleErrorWith { _ => + httpSigChallenge(url, lastReq, failed, nel) } yield authdReq case _ => ??? // fail From b6ed705de75c0becc64b2883d39dcc1643289a00 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Thu, 9 Mar 2023 16:37:59 +0100 Subject: [PATCH 02/42] Develop LDES example tests --- .../src/main/scala/run/cosy/ld/PNGraph.scala | 173 ++++++++++------- .../test/scala/run/cosy/ld/JenaWebTest.scala | 2 + .../scala/run/cosy/ld/LdesSimpleWebTest.scala | 148 +++++++++++++++ .../test/scala/run/cosy/ld/LdesWebTest.scala | 16 -- .../scala/run/cosy/ld/ldes/MiniLdesWWW.scala | 174 +++++++++++------- project/Dependencies.scala | 3 +- 6 files changed, 363 insertions(+), 153 deletions(-) create mode 100644 ldes/shared/src/test/scala/run/cosy/ld/LdesSimpleWebTest.scala delete mode 100644 ldes/shared/src/test/scala/run/cosy/ld/LdesWebTest.scala diff --git a/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala b/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala index 0088daf..52e22ba 100644 --- a/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala +++ b/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala @@ -9,45 +9,66 @@ import org.w3.banana.{Ops, RDF} trait Web[F[_]: Concurrent, R <: RDF]: /** get a Graph for the given URL (without a fragment) */ def get(url: RDF.URI[R]): F[RDF.Graph[R]] + /** get Pointed Named Graph for given url */ def getPNG(url: RDF.URI[R]): F[UriNGraph[R]] - /** A Pointed Named Graph, ie, a pointer into a NamedGraph We don't use a case class here as * equality between PNGgraphs is complicated by the need to prove isomorphism between graphs, and * the nodes have to be equivalent. - * - * todo: it would make sense to have subtypes for different types of points, so - * that filters on points that are urls can be typed. Also when the point is on the - * literal then / makes less sense. png.jump makes sense only if the node is a URL + * + * todo: it would make sense to have subtypes for different types of points, so that filters on + * points that are urls can be typed. Also when the point is on the literal then / makes less + * sense. png.jump makes sense only if the node is a URL */ trait PNGraph[R <: RDF]: - type Point <: Node[R] + type Point <: RDF.URI[R] | RDF.Literal[R] | RDF.BNode[R] def point: Point def name: URI[R] def graph: Graph[R] - + + /** collect from the given point relations forward and backward (where possible) and return a + * graph + */ + def collect( + forward: RDF.URI[R]* + )( + backward: RDF.URI[R]* + )(using ops: Ops[R]): RDF.Graph[R] = + import ops.{*, given} + val f: Seq[Seq[RDF.Triple[R]]] = + if point.isURI || point.isBNode then + for frel <- forward + yield graph.find(point, frel, `*`).toSeq + else Seq.empty + val b: Seq[Seq[RDF.Triple[R]]] = + for brel <- backward + yield graph.find(`*`, brel, point).toSeq + Graph(f.flatten ++ b.flatten) + trait SubjPNGraph[R <: RDF] extends PNGraph[R]: type Point <: RDF.Statement.Subject[R] - + def /(rel: RDF.URI[R])(using ops: Ops[R]): Seq[PNGraph[R]] = - import ops.{*, given} - graph.find(point, rel, `*`) - .map { (tr: RDF.Triple[R]) => - tr.obj.asNode.fold( - uri => new UriNGraph(uri,name,graph), - bn => new BNodeNGraph[R](bn, name, graph), - lit => new LiteralNGraph[R](lit, name, graph) - ) - } - .toSeq + import ops.{*, given} + graph + .find(point, rel, `*`) + .map { (tr: RDF.Triple[R]) => + tr.obj.asNode.fold( + uri => new UriNGraph(uri, name, graph), + bn => new BNodeNGraph[R](bn, name, graph), + lit => new LiteralNGraph[R](lit, name, graph) + ) + } + .toSeq end / - + inline def rel(rel: RDF.URI[R])(using ops: Ops[R]): Seq[PNGraph[R]] = /(rel) - + def \(rel: RDF.URI[R])(using ops: Ops[R]): Seq[PNGraph[R]] = import ops.{*, given} - graph.find(`*`, rel, point) + graph + .find(`*`, rel, point) .map { (tr: RDF.Triple[R]) => val s: Statement.Subject[R] = tr.subj s.foldSubj[PNGraph[R]]( @@ -57,78 +78,82 @@ trait SubjPNGraph[R <: RDF] extends PNGraph[R]: } .toSeq end \ - -end SubjPNGraph + def hasType(tp: RDF.URI[R])(using ops: Ops[R]): Boolean = + import ops.{*, given} + val it: Iterator[RDF.Triple[R]] = graph.find(point, rdf.typ, tp) + it.hasNext + +end SubjPNGraph /** A PNG where the point is a URI */ class UriNGraph[R <: RDF]( - val point: URI[R], - val name: URI[R], - val graph: Graph[R] -)(using ops: Ops[R]) extends SubjPNGraph[R]: - import ops.{*,given} - type Point = RDF.URI[R] - - lazy val pointLoc: RDF.URI[R] = point.fragmentLess - lazy val isLocal: Boolean = pointLoc == name - + val point: URI[R], + val name: URI[R], + val graph: Graph[R] +)(using ops: Ops[R]) + extends SubjPNGraph[R]: + import ops.{*, given} + type Point = RDF.URI[R] + + lazy val pointLoc: RDF.URI[R] = point.fragmentLess + lazy val isLocal: Boolean = pointLoc == name + //todo: things are more complex when one takes redirects into account... - def jump[F[_]: Concurrent](using web: Web[F, R]): F[UriNGraph[R]] = - import cats.syntax.functor.* - import ops.{*, given} - if isLocal then summon[Concurrent[F]].pure(this) - else web.get(pointLoc).map(g => new UriNGraph(point, pointLoc, g)) - -end UriNGraph + def jump[F[_]: Concurrent](using web: Web[F, R]): F[UriNGraph[R]] = + import cats.syntax.functor.* + import ops.{*, given} + if isLocal then summon[Concurrent[F]].pure(this) + else web.get(pointLoc).map(g => new UriNGraph(point, pointLoc, g)) +end UriNGraph /** A PNG where the point is a BNode */ class BNodeNGraph[R <: RDF]( - val point: RDF.BNode[R], - val name: URI[R], - val graph: Graph[R] + val point: RDF.BNode[R], + val name: URI[R], + val graph: Graph[R] ) extends SubjPNGraph[R]: - type Point = RDF.BNode[R] - + type Point = RDF.BNode[R] /** A PNG where the point is a BNode */ class LiteralNGraph[R <: RDF]( - val point: RDF.Literal[R], - val name: URI[R], - val graph: Graph[R] + val point: RDF.Literal[R], + val name: URI[R], + val graph: Graph[R] ) extends PNGraph[R]: type Point = RDF.Literal[R] - //todo: this is duplicated code with SubjPNGraph. + // todo: this is duplicated code with SubjPNGraph. def \(rel: RDF.URI[R])(using ops: Ops[R]): Seq[PNGraph[R]] = import ops.{*, given} - graph.find(`*`, rel, point) + graph + .find(`*`, rel, point) .map { (tr: RDF.Triple[R]) => tr.subj.foldSubj( uri => UriNGraph(uri, name, graph), bn => BNodeNGraph(bn, name, graph) ) - }.toSeq + } + .toSeq end LiteralNGraph - object PNGraph: - + extension [F[_]: cats.effect.Concurrent, R <: RDF]( - pngs: Seq[PNGraph[R]] + pngs: Seq[PNGraph[R]] )(using web: Web[F, R], ops: Ops[R]) - - def jump : fs2.Stream[F, PNGraph[R]] = + // jump on nodes in the sequence that can be jumped + def jump: fs2.Stream[F, PNGraph[R]] = import ops.given var local: List[PNGraph[R]] = Nil var remoteGs: List[UriNGraph[R]] = Nil pngs.foreach { - case ug: UriNGraph[R] if !ug.isLocal => remoteGs = ug :: remoteGs - case other => local = other::local + case ug: UriNGraph[R] if !ug.isLocal => remoteGs = ug :: remoteGs + case other => local = other :: local } import cats.syntax.all.given - + val in: fs2.Stream[Pure, PNGraph[R]] = fs2.Stream.chunk(fs2.Chunk.seq(local)) val out: fs2.Stream[F, PNGraph[R]] = fs2.Stream @@ -140,18 +165,36 @@ object PNGraph: web.get(remote.pointLoc).map(g => UriNGraph[R](remote.point, remote.pointLoc, g)) } in ++ out - end jump + def rel(relation: RDF.URI[R]): Seq[PNGraph[R]] = + pngs.collect { case sub: SubjPNGraph[R] => + sub / relation + }.flatten - extension[F[_] : cats.effect.Concurrent, R <: RDF] ( - pngs: fs2.Stream[F, PNGraph[R]] + def filterType(tp: RDF.URI[R]): Seq[SubjPNGraph[R]] = + pngs.collect { + case sub: SubjPNGraph[R] if sub.hasType(tp) => sub + } + + extension [F[_]: cats.effect.Concurrent, R <: RDF]( + pngs: fs2.Stream[F, PNGraph[R]] )(using ops: Ops[R]) - + def /(rel: RDF.URI[R]): fs2.Stream[F, PNGraph[R]] = - pngs.collect{ - case sub: SubjPNGraph[R] => fs2.Stream(sub/rel*) + pngs.collect { case sub: SubjPNGraph[R] => + fs2.Stream(sub / rel*) }.flatten inline def rel(rel: RDF.URI[R]): fs2.Stream[F, PNGraph[R]] = /(rel) + def jump(using Web[F, R]): fs2.Stream[F, SubjPNGraph[R]] = + pngs.collect { + case bn: BNodeNGraph[R] => fs2.Stream(bn) + case un: UriNGraph[R] => fs2.Stream.eval(un.jump) + }.flatten + + def filterType(tp: RDF.URI[R]): fs2.Stream[F, SubjPNGraph[R]] = + pngs.collect { + case sub: SubjPNGraph[R] if sub.hasType(tp) => sub + } diff --git a/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala b/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala index bee9bea..a51b0e0 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala @@ -5,3 +5,5 @@ import org.w3.banana.jena.JenaRdf.{R, given} class JenaFoafWebTest extends FoafWebTest[R]() +class JenaLdesSimpleWebTest extends LdesSimpleWebTest[R]() + diff --git a/ldes/shared/src/test/scala/run/cosy/ld/LdesSimpleWebTest.scala b/ldes/shared/src/test/scala/run/cosy/ld/LdesSimpleWebTest.scala new file mode 100644 index 0000000..67a5a74 --- /dev/null +++ b/ldes/shared/src/test/scala/run/cosy/ld/LdesSimpleWebTest.scala @@ -0,0 +1,148 @@ +package run.cosy.ld + +import cats.effect.IO +import cats.effect.kernel.Concurrent +import fs2.Chunk +import io.lemonlabs as ll +import munit.CatsEffectSuite +import org.w3.banana.{Ops, RDF} +import run.cosy.ld.ldes.MiniLdesWWW + +/** We deal here with simple data that does not have link loops or such problems We will need to + * work on that in separate tests. Here we just want to test how to get the data + */ +trait LdesSimpleWebTest[R <: RDF]()(using ops: Ops[R]) extends CatsEffectSuite: + import MiniLdesWWW.* + val miniWeb = new MiniLdesWWW[R] + given www: Web[IO, R] = miniWeb + import miniWeb.{sosa, tree, wgs84} + import ops.{*, given} + import run.cosy.ld.PNGraph.* + + test("walk through pages starting from first page") { + // we start from the container url, and jump to the first page + val page1_IO: IO[UriNGraph[R]] = www.getPNG(URI(D09_05)) + + // nodeStr has to be in the right graph + def next(node: SubjPNGraph[R]): Seq[PNGraph[R]] = + node + .rel(tree.relation) + .filterType(tree.GreaterThanRelation) + .rel(tree.node) + + val result: IO[fs2.Stream[IO, String]] = + for page1 <- page1_IO + yield + // this is a bit clumsy but it works + fs2.Stream(page1.name.value) + ++ fs2.Stream.unfoldEval(page1) { (node: UriNGraph[R]) => + val ns: Seq[PNGraph[R]] = next(node) + val x: Option[IO[UriNGraph[R]]] = ns.collectFirst { case ui: UriNGraph[R] => + ui.jump + } + x match + case None => IO(None) + case Some(ioNgr) => ioNgr.map(ngr => Some((ngr.name.value, ngr))) + } + result.flatMap(names => + names.compile.toList.map { lst => + assertEquals(lst.toSet, Set(D09_05, D09_06, D09_07)) + } + ) + } + + test("walk through pages but starting from Collection") { + val z: IO[fs2.Stream[IO, String]] = + for views <- www.getPNG(URI(Collection)).map(_.rel(tree.view)) + yield fs2.Stream + .unfoldLoopEval(views) { (views: Seq[PNGraph[R]]) => + import cats.syntax.traverse.{*, given} + val x: Seq[IO[UriNGraph[R]]] = + // because we are using unfoldLoopEval we need to work with the IO effect, not Streams + // so we cannot just use views.jump + views.collect { case ung: UriNGraph[R] => ung.jump } + val y: IO[Seq[UriNGraph[R]]] = x.sequence + val res: IO[(Chunk[String], Option[Seq[UriNGraph[R]]])] = y.map { seqUng => + val s: Seq[UriNGraph[R]] = seqUng + .rel(tree.relation) + .filterType(tree.GreaterThanRelation) + .rel(tree.node) + .collect { case ung: UriNGraph[R] => ung } + ( + Chunk.seq(views.collect { case p: UriNGraph[R] => p.point.value }), + if s.isEmpty then None + else Some(s) + ) + } + res + } + .unchunks + z.flatMap { urls => + urls.compile.toList.map { lst => + assertEquals(lst.toSet, Set(D09_05, D09_06, D09_07)) + } + } + } + + test("walk through pages and collect observations") { + val z: IO[fs2.Stream[IO, RDF.Graph[R]]] = + for views <- www.getPNG(URI(Collection)).map(_.rel(tree.view)) + yield fs2.Stream + .unfoldLoopEval(views) { (views: Seq[PNGraph[R]]) => + import cats.syntax.traverse.{*, given} + val x: Seq[IO[UriNGraph[R]]] = + views.collect { case ung: UriNGraph[R] => ung.jump } + val pagesIO: IO[Seq[UriNGraph[R]]] = x.sequence +// val pagesIO: IO[Seq[UriNGraph[R]]] = +// views.collect({ case ung: UriNGraph[R] => ung.jump }).sequence + val res: IO[(RDF.Graph[R], Option[Seq[UriNGraph[R]]])] = pagesIO.map { seqUng => + val nextPages: Seq[UriNGraph[R]] = + seqUng + .rel(tree.relation) + .filterType(tree.GreaterThanRelation) + .rel(tree.node) + .collect { case ung: UriNGraph[R] => ung } + // we need to place the pointer on the Collection of each page + val collInPages: Seq[UriNGraph[R]] = + val uc = URI(Collection) + seqUng.map(ung => new UriNGraph[R](uc, ung.name, ung.graph)) + val obs = collInPages + .rel(tree.member) + .map(png => + png.collect( + rdf.typ, + wgs84.location, + sosa.hasSimpleResult, + sosa.madeBySensor, + sosa.observedProperty, + sosa.resultTime + )() + ) + .fold(Graph.empty)((g1, g2) => g1 union g2) + ( + obs, + if nextPages.isEmpty then None + else Some(nextPages) + ) + } + res + } + z.flatMap { graphs => + graphs.compile.toList.map { lst => + val g: RDF.Graph[R] = lst.fold(Graph.empty)((g1, g2) => g1 union g2) + val gTrpls: Set[RDF.Triple[R]] = Set(g.triples.toSeq*) + val expectedTrpls: Seq[RDF.Triple[R]] = miniWeb.obsrvs.iterator + .map { (uristr, rtriples) => + val u = ll.uri.AbsoluteUrl.parse(uristr) + rtriples.flatten.map((rt: RDF.rTriple[R]) => rt.resolveAgainst(u)._1) + } + .toSeq + .flatten + val expected: Set[RDF.Triple[R]] = Set(expectedTrpls*) + // we can compare them as sets as we have no blank nodes + assertEquals(expected.diff(gTrpls), Set.empty) + assertEquals(gTrpls.size, expected.size) + assertEquals(gTrpls, expected) + } + } + } diff --git a/ldes/shared/src/test/scala/run/cosy/ld/LdesWebTest.scala b/ldes/shared/src/test/scala/run/cosy/ld/LdesWebTest.scala deleted file mode 100644 index 420bf2a..0000000 --- a/ldes/shared/src/test/scala/run/cosy/ld/LdesWebTest.scala +++ /dev/null @@ -1,16 +0,0 @@ -package run.cosy.ld -import cats.effect.IO -import org.w3.banana.RDF -import org.w3.banana.Ops -import run.cosy.ld.ldes.MiniLdesWWW -import munit.CatsEffectSuite - -trait LdesWebTest[R <: RDF]()(using ops: Ops[R]) extends CatsEffectSuite: - - val miniWeb = new MiniLdesWWW[R] - import miniWeb.foaf - given www: Web[IO, R] = miniWeb - - test("") { - - } diff --git a/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala b/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala index f2b6d45..63a8f4d 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala @@ -6,13 +6,14 @@ import org.w3.banana.diesel.{*, given} import org.w3.banana.{diesel, *} import run.cosy.ld.* import run.cosy.ld.ldes.prefix as ldesPre +import scala.language.implicitConversions object MiniLdesWWW: def mechelen(date: String): String = "https://ldes.mechelen.org/" + date val D09_05: String = mechelen("2021-09-05") val D09_06: String = mechelen("2021-09-06") val D09_07: String = mechelen("2021-09-07") - val Container: String = mechelen("") + val Collection: String = mechelen("") class MiniLdesWWW[R <: RDF](using ops: Ops[R]) extends Web[IO, R]: import MiniLdesWWW.* @@ -33,16 +34,110 @@ class MiniLdesWWW[R <: RDF](using ops: Ops[R]) extends Web[IO, R]: val doc = url.fragmentLess get(doc).map(g => new UriNGraph(url, doc, g)) + def observation( + name: String, + loc: String, + simpleResult: String, + sensor: RDF.URI[R], + observedProp: RDF.URI[R], + resultTime: String + ): Seq[RDF.rTriple[R]] = + (rURI("#" + name) -- rdf.typ ->- sosa.Observation + -- wgs84.location ->- area(loc) + -- sosa.hasSimpleResult ->- (simpleResult ^^ xsd.float) + -- sosa.madeBySensor ->- sensor + -- sosa.observedProperty ->- observedProp + -- sosa.resultTime ->- (resultTime ^^ xsd.dateTimeStamp)).graph.triples.toSeq + + val obsrvs: Map[String, Seq[Seq[RDF.rTriple[R]]]] = Map( + D09_05 -> Seq( + observation( + "3", + "loc781089", + "4.0", + pzDev("213501"), + polMsr("motorized"), + "2021-09-05T23:00:00+02" + ), + observation( + "482", + "loc", + "2455.1123", + crop("schoolstraat"), + cropProp("deviceNbr"), + "2021-09-05T22:30:00+02" + ), + observation( + "4464", + "loc734383", + "10.0", + pzDev("213504+5+6"), + polMsr("bike"), + "2021-09-05T23:00:00+02" + ) + ), + D09_06 -> Seq( + observation( + "3003", + "loc763628", + "44.0", + pzDev("213503"), + polMsr("motorized"), + "2021-09-06T11:00:00+02" + ), + observation( + "4493", + "loc734383", + "197.0", + pzDev("213504+5+6"), + polMsr("motorized"), + "2021-09-06T12:00:00+02" + ), + observation( + "48", + "loc781089", + "1.0", + pzDev("213501"), + polMsr("bike"), + "2021-09-06T22:00:00+02" + ) + ), + D09_07 -> Seq( + observation( + "658", + "loc", + "5087.4795", + crop("schoolstraat"), + cropProp("deviceNbr"), + "2021-09-07T18:30:00+02" + ), + observation( + "637", + "loc", + "7009.3345", + crop("schoolstraat"), + cropProp("deviceNbr"), + "2021-09-07T13:15:00+02" + ), + observation( + "3074", + "loc763628", + "1.0", + pzDev("213503"), + polMsr("bike"), + "2021-09-06T22:00:00+02" + ) + ) + ) + def get(url: RDF.URI[R]): IO[RDF.Graph[R]] = - import scala.language.implicitConversions val res: RDF.rGraph[R] = url.value match - case Container => + case Collection => (rURI("").a(ldes.EventStream) -- ldes.timestampPath ->- sosa.resultTime -- tree.shape ->- rURI("flows-shacl") - -- tree.view ->- rURI("2021-09-05") - ).graph + -- tree.view ->- rURI("2021-09-05")).graph case D09_05 => (rURI("") -- rdf.typ ->- tree.Node -- tree.relation ->- ( @@ -50,28 +145,7 @@ class MiniLdesWWW[R <: RDF](using ops: Ops[R]) extends Web[IO, R]: -- tree.node ->- rURI("2021-09-06") -- tree.path ->- sosa.resultTime -- tree.value ->- ("2021-09-06T00:00:00+02" ^^ xsd.dateTimeStamp) - )).graph ++ ( - rURI("#3") -- rdf.typ ->- sosa.Observation - -- wgs84.location ->- area("loc781089") - -- sosa.hasSimpleResult ->- ("4.0" ^^ xsd.float) - -- sosa.madeBySensor ->- pzDev("213501") - -- sosa.observedProperty ->- polMsr("motorized") - -- sosa.resultTime ->- ("2021-09-05T23:00:00+02" ^^ xsd.dateTimeStamp) - ).graph.triples.toSeq ++ ( - rURI("#482") -- rdf.typ ->- sosa.Observation - -- wgs84.location ->- area("loc") - -- sosa.hasSimpleResult ->- ("2455.1123" ^^ xsd.float) - -- sosa.madeBySensor ->- crop("schoolstraat") - -- sosa.observedProperty ->- cropProp("deviceNbr") - -- sosa.resultTime ->- ("2021-09-05T22:30:00+02" ^^ xsd.dateTimeStamp) - ).graph.triples.toSeq ++ ( - rURI("#4464") -- rdf.typ ->- sosa.Observation - -- wgs84.location ->- area("loc734383") - -- sosa.hasSimpleResult ->- ("10.0" ^^ xsd.float) - -- sosa.madeBySensor ->- pzDev("213504+5+6") - -- sosa.observedProperty ->- polMsr("bike") - -- sosa.resultTime ->- ("2021-09-05T23:00:00+02" ^^ xsd.dateTimeStamp) - ).graph.triples.toSeq ++ ( + )).graph ++ obsrvs(D09_05).flatten ++ ( rURI(".").a(ldes.EventStream) -- ldes.timestampPath ->- sosa.resultTime -- tree.shape ->- rURI("flows-shacl") @@ -93,28 +167,7 @@ class MiniLdesWWW[R <: RDF](using ops: Ops[R]) extends Web[IO, R]: -- tree.node ->- rURI("2021-09-07") -- tree.path ->- sosa.resultTime -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) - )).graph ++ ( - rURI("#3003").a(sosa.Observation) - -- wgs84.location ->- area("loc763628") - -- sosa.hasSimpleResult ->- ("44.0" ^^ xsd.float) - -- sosa.madeBySensor ->- pzDev("213503") - -- sosa.observedProperty ->- polMsr("motorized") - -- sosa.resultTime ->- ("2021-09-06T11:00:00+02" ^^ xsd.dateTimeStamp) - ).graph.triples.toSeq ++ ( - rURI("#4493").a(sosa.Observation) - -- wgs84.location ->- area("loc734383") - -- sosa.hasSimpleResult ->- ("197.0" ^^ xsd.float) - -- sosa.madeBySensor ->- pzDev("213504+5+6") - -- sosa.observedProperty ->- polMsr("motorized") - -- sosa.resultTime ->- ("2021-09-06T12:00:00+02" ^^ xsd.dateTimeStamp) - ).graph.triples.toSeq ++ ( - rURI("#48").a(sosa.Observation) - -- wgs84.location ->- area("loc781089") - -- sosa.hasSimpleResult ->- ("1.0" ^^ xsd.float) - -- sosa.madeBySensor ->- pzDev("213501") - -- sosa.observedProperty ->- polMsr("bike") - -- sosa.resultTime ->- ("2021-09-06T22:00:00+02" ^^ xsd.dateTimeStamp) - ).graph.triples.toSeq ++ ( + )).graph ++ obsrvs(D09_06).flatten ++ ( rURI(".").a(ldes.EventStream) -- ldes.timestampPath ->- sosa.resultTime -- tree.shape ->- rURI("flows-shacl") @@ -130,28 +183,7 @@ class MiniLdesWWW[R <: RDF](using ops: Ops[R]) extends Web[IO, R]: -- tree.node ->- rURI("2021-09-07") -- tree.path ->- sosa.resultTime -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) - )).graph ++ ( - rURI("#658").a(sosa.Observation) - -- wgs84.location ->- area("loc") - -- sosa.hasSimpleResult ->- ("5087.4795" ^^ xsd.float) - -- sosa.madeBySensor ->- crop("schoolstraat") - -- sosa.observedProperty ->- cropProp("deviceNbr") - -- sosa.resultTime ->- ("2021-09-07T18:30:00+02" ^^ xsd.dateTimeStamp) - ).graph.triples.toSeq ++ ( - rURI("#637").a(sosa.Observation) - -- wgs84.location ->- area("loc") - -- sosa.hasSimpleResult ->- ("7009.3345" ^^ xsd.float) - -- sosa.madeBySensor ->- crop("schoolstraat") - -- sosa.observedProperty ->- cropProp("deviceNbr") - -- sosa.resultTime ->- ("2021-09-07T13:15:00+02" ^^ xsd.dateTimeStamp) - ).graph.triples.toSeq ++ ( - rURI("#3074").a(sosa.Observation) - -- wgs84.location ->- area("loc763628") - -- sosa.hasSimpleResult ->- ("1.0" ^^ xsd.float) - -- sosa.madeBySensor ->- pzDev("213503") - -- sosa.observedProperty ->- polMsr("bike") - -- sosa.resultTime ->- ("2021-09-06T22:00:00+02" ^^ xsd.dateTimeStamp) - ).graph.triples.toSeq ++ ( + )).graph ++ obsrvs(D09_07).flatten ++ ( rURI(".").a(ldes.EventStream) -- ldes.timestampPath ->- sosa.resultTime -- tree.shape ->- rURI("flows-shacl") diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 3c303dd..b99c04e 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -5,7 +5,7 @@ object Dependencies { object Ver { val scala = "3.2.2" val http4s = "1.0.0-M39" - val banana = "0.9-79e8845-20230228T213613Z-SNAPSHOT" + val banana = "0.9-c996591-SNAPSHOT" val bobcats = "0.3-3236e64-SNAPSHOT" val httpSig = "0.4-ac23f8b-SNAPSHOT" } @@ -33,6 +33,7 @@ object Dependencies { lazy val core = Def.setting("org.typelevel" %%% "cats-core" % "2.9.0") lazy val free = Def.setting("org.typelevel" %%% "cats-free" % "2.9.0") // lazy val effect = Def.setting("org.typelevel" %% "cats-effect" % "3.4.8") + // https://github.com/typelevel/fs2 lazy val fs2 = Def.setting("co.fs2" %% "fs2-core" % "3.6.1") // https://github.com/typelevel/munit-cats-effect From a17e9aa8fe8dff08a6f13948916718f2f4a71d44 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Fri, 10 Mar 2023 19:07:12 +0100 Subject: [PATCH 03/42] added an example with a circularity and a test for it --- .../test/scala/run/cosy/ld/JenaWebTest.scala | 3 + .../run/cosy/ld/LdesCircularWebTest.scala | 93 ++++++++++++++ .../scala/run/cosy/ld/LdesSimpleWebTest.scala | 2 + .../cosy/ld/ldes/CircularMiniLdesWWW.scala | 37 ++++++ .../scala/run/cosy/ld/ldes/MiniLdesWWW.scala | 121 +++++++++--------- 5 files changed, 197 insertions(+), 59 deletions(-) create mode 100644 ldes/shared/src/test/scala/run/cosy/ld/LdesCircularWebTest.scala create mode 100644 ldes/shared/src/test/scala/run/cosy/ld/ldes/CircularMiniLdesWWW.scala diff --git a/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala b/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala index a51b0e0..6cfea06 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala @@ -7,3 +7,6 @@ class JenaFoafWebTest extends FoafWebTest[R]() class JenaLdesSimpleWebTest extends LdesSimpleWebTest[R]() +class JenaLdesCircularWebTest extends LdesCircularWebTest[R]() + + diff --git a/ldes/shared/src/test/scala/run/cosy/ld/LdesCircularWebTest.scala b/ldes/shared/src/test/scala/run/cosy/ld/LdesCircularWebTest.scala new file mode 100644 index 0000000..0d87720 --- /dev/null +++ b/ldes/shared/src/test/scala/run/cosy/ld/LdesCircularWebTest.scala @@ -0,0 +1,93 @@ +package run.cosy.ld + +import cats.effect.IO +import cats.effect.kernel.{Concurrent, Ref} +import io.lemonlabs as ll +import munit.CatsEffectSuite +import org.w3.banana.{Ops, RDF} +import run.cosy.ld.ldes.{CircularMiniLdesWWW, MiniLdesWWW} + +import scala.collection.immutable.{Seq, Set} + +trait LdesCircularWebTest[R <: RDF]()(using ops: Ops[R]) extends CatsEffectSuite: + import MiniLdesWWW.* + val miniWeb = new CircularMiniLdesWWW[R] + given www: Web[IO, R] = miniWeb + import miniWeb.{sosa, tree, wgs84} + import ops.{*, given} + import run.cosy.ld.PNGraph.* + + test("get circular data but recognise circularity") { + import cats.syntax.traverse.{*, given} + val z: IO[fs2.Stream[IO, RDF.Graph[R]]] = { + for { + visitedRef <- Ref.of[IO, Set[RDF.URI[R]]](Set()) + views <- www.getPNG(URI(Collection)).map(_.rel(tree.view)) + } yield { + fs2.Stream + .unfoldLoopEval(views) { views => + import cats.syntax.traverse.{*, given} + for + v <- visitedRef.get + // here we make sure we don't visit the same page twice. + pages <- views.collect { + case ung: UriNGraph[R] if !v.contains(ung.point.fragmentLess) => ung.jump[IO] + }.sequence + _ <- visitedRef.update { v => + val urls = pages.map(_.name) + v.union(urls.toSet) + } + yield { + val nextPages: Seq[UriNGraph[R]] = pages + .asInstanceOf[Seq[UriNGraph[R]]] + .rel(tree.relation) + .filterType(tree.GreaterThanRelation) + .rel(tree.node) + .collect { case ung: UriNGraph[R] => ung } + // we need to place the pointer on the Collection of each page + val collInPages: Seq[UriNGraph[R]] = + val uc = URI(Collection) + pages.map(ung => new UriNGraph[R](uc, ung.name, ung.graph)) + val obs: RDF.Graph[R] = collInPages + .rel(tree.member) + .map(png => + png.collect( + rdf.typ, + wgs84.location, + sosa.hasSimpleResult, + sosa.madeBySensor, + sosa.observedProperty, + sosa.resultTime + )() + ) + .fold(Graph.empty)((g1, g2) => g1 union g2) + ( + obs, + if nextPages.isEmpty then None + else Some(nextPages) + ) + } + + } + } + } + + z.flatMap { graphs => + graphs.compile.toList.map { lst => + val g: RDF.Graph[R] = lst.fold(Graph.empty)((g1, g2) => g1 union g2) + val gTrpls: Set[RDF.Triple[R]] = Set(g.triples.toSeq*) + val expectedTrpls: Seq[RDF.Triple[R]] = miniWeb.obsrvs.iterator + .map { (uristr, rtriples) => + val u = ll.uri.AbsoluteUrl.parse(uristr) + rtriples.flatten.map((rt: RDF.rTriple[R]) => rt.resolveAgainst(u)._1) + } + .toSeq + .flatten + val expected: Set[RDF.Triple[R]] = Set(expectedTrpls*) + // we can compare them as sets as we have no blank nodes + assertEquals(expected.diff(gTrpls), Set.empty) + assertEquals(gTrpls.size, expected.size) + assertEquals(gTrpls, expected) + } + } + } diff --git a/ldes/shared/src/test/scala/run/cosy/ld/LdesSimpleWebTest.scala b/ldes/shared/src/test/scala/run/cosy/ld/LdesSimpleWebTest.scala index 67a5a74..db4cac9 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/LdesSimpleWebTest.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/LdesSimpleWebTest.scala @@ -92,6 +92,8 @@ trait LdesSimpleWebTest[R <: RDF]()(using ops: Ops[R]) extends CatsEffectSuite: import cats.syntax.traverse.{*, given} val x: Seq[IO[UriNGraph[R]]] = views.collect { case ung: UriNGraph[R] => ung.jump } + //note: a problem with x.sequence is that it would need all UriNGraphs to be complete + //before the IO is complete. What if some get stuck? val pagesIO: IO[Seq[UriNGraph[R]]] = x.sequence // val pagesIO: IO[Seq[UriNGraph[R]]] = // views.collect({ case ung: UriNGraph[R] => ung.jump }).sequence diff --git a/ldes/shared/src/test/scala/run/cosy/ld/ldes/CircularMiniLdesWWW.scala b/ldes/shared/src/test/scala/run/cosy/ld/ldes/CircularMiniLdesWWW.scala new file mode 100644 index 0000000..fbda701 --- /dev/null +++ b/ldes/shared/src/test/scala/run/cosy/ld/ldes/CircularMiniLdesWWW.scala @@ -0,0 +1,37 @@ +package run.cosy.ld.ldes + +import cats.effect.IO +import org.w3.banana.* +import org.w3.banana.diesel.{*, given} + +class CircularMiniLdesWWW[R <: RDF](using ops: Ops[R]) extends MiniLdesWWW[R]: + import MiniLdesWWW.* + import ops.{*, given} + + // we introduce a circularity in the pagination of D09_07 + override def getRelativeGraph(url: RDF.URI[R]): RDF.rGraph[R] = + import scala.language.implicitConversions + url.value match + case D09_07 => + (rURI("").a(tree.Node) + -- tree.relation ->- ( + BNode().a(tree.LessThanRelation) + -- tree.node ->- rURI("2021-09-07") + -- tree.path ->- sosa.resultTime + -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) + ) + -- tree.relation ->- ( + BNode().a(tree.GreaterThanRelation) + -- tree.node ->- rURI("2021-09-05") //warning: error circularity + -- tree.path ->- sosa.resultTime + -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) + )).graph ++ obsrvs(D09_07).flatten ++ ( + rURI(".").a(ldes.EventStream) + -- ldes.timestampPath ->- sosa.resultTime + -- tree.shape ->- rURI("flows-shacl") + -- tree.view ->- rURI("") + -- tree.member ->- rURI("#658") + -- tree.member ->- rURI("#3074") + -- tree.member ->- rURI("#637") + ).graph.triples.toSeq + case _ => super.getRelativeGraph(url) diff --git a/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala b/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala index 63a8f4d..c1e0a2a 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala @@ -30,6 +30,10 @@ class MiniLdesWWW[R <: RDF](using ops: Ops[R]) extends Web[IO, R]: def crop(area: String) = URI("https://data.cropland.be/area#" + area) def cropProp(prop: String) = URI("https://data.cropland.be/measure#" + prop) + def get(url: RDF.URI[R]): IO[RDF.Graph[R]] = + val res: RDF.rGraph[R] = getRelativeGraph(url) + IO(res.resolveAgainst(AbsoluteUrl.parse(url.value))) + def getPNG(url: RDF.URI[R]): IO[UriNGraph[R]] = val doc = url.fragmentLess get(doc).map(g => new UriNGraph(url, doc, g)) @@ -129,67 +133,66 @@ class MiniLdesWWW[R <: RDF](using ops: Ops[R]) extends Web[IO, R]: ) ) ) - - def get(url: RDF.URI[R]): IO[RDF.Graph[R]] = - val res: RDF.rGraph[R] = - url.value match - case Collection => - (rURI("").a(ldes.EventStream) + + def getRelativeGraph(url: RDF.URI[R]): RDF.rGraph[R] = + url.value match + case Collection => + (rURI("").a(ldes.EventStream) + -- ldes.timestampPath ->- sosa.resultTime + -- tree.shape ->- rURI("flows-shacl") + -- tree.view ->- rURI("2021-09-05")).graph + case D09_05 => + (rURI("") -- rdf.typ ->- tree.Node + -- tree.relation ->- ( + BNode() -- rdf.typ ->- tree.GreaterThanRelation + -- tree.node ->- rURI("2021-09-06") + -- tree.path ->- sosa.resultTime + -- tree.value ->- ("2021-09-06T00:00:00+02" ^^ xsd.dateTimeStamp) + )).graph ++ obsrvs(D09_05).flatten ++ ( + rURI(".").a(ldes.EventStream) -- ldes.timestampPath ->- sosa.resultTime -- tree.shape ->- rURI("flows-shacl") - -- tree.view ->- rURI("2021-09-05")).graph - case D09_05 => - (rURI("") -- rdf.typ ->- tree.Node - -- tree.relation ->- ( - BNode() -- rdf.typ ->- tree.GreaterThanRelation - -- tree.node ->- rURI("2021-09-06") - -- tree.path ->- sosa.resultTime - -- tree.value ->- ("2021-09-06T00:00:00+02" ^^ xsd.dateTimeStamp) - )).graph ++ obsrvs(D09_05).flatten ++ ( - rURI(".").a(ldes.EventStream) - -- ldes.timestampPath ->- sosa.resultTime - -- tree.shape ->- rURI("flows-shacl") - -- tree.view ->- rURI("") - -- tree.member ->- rURI("#3") - -- tree.member ->- rURI("#482") - -- tree.member ->- rURI("#4464") + -- tree.view ->- rURI("") + -- tree.member ->- rURI("#3") + -- tree.member ->- rURI("#482") + -- tree.member ->- rURI("#4464") ).graph.triples.toSeq - case D09_06 => - (rURI("").a(tree.Node) - -- tree.relation ->- ( - BNode().a(tree.LessThanRelation) - -- tree.node ->- rURI("2021-09-05") - -- tree.path ->- sosa.resultTime - -- tree.value ->- ("2021-09-06T00:00:00+02" ^^ xsd.dateTimeStamp) - ) - -- tree.relation ->- ( - BNode().a(tree.GreaterThanRelation) - -- tree.node ->- rURI("2021-09-07") - -- tree.path ->- sosa.resultTime - -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) - )).graph ++ obsrvs(D09_06).flatten ++ ( - rURI(".").a(ldes.EventStream) - -- ldes.timestampPath ->- sosa.resultTime - -- tree.shape ->- rURI("flows-shacl") - -- tree.view ->- rURI("") - -- tree.member ->- rURI("#4493") - -- tree.member ->- rURI("#48") - -- tree.member ->- rURI("#3003") + case D09_06 => + (rURI("").a(tree.Node) + -- tree.relation ->- ( + BNode().a(tree.LessThanRelation) + -- tree.node ->- rURI("2021-09-05") + -- tree.path ->- sosa.resultTime + -- tree.value ->- ("2021-09-06T00:00:00+02" ^^ xsd.dateTimeStamp) + ) + -- tree.relation ->- ( + BNode().a(tree.GreaterThanRelation) + -- tree.node ->- rURI("2021-09-07") + -- tree.path ->- sosa.resultTime + -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) + )).graph ++ obsrvs(D09_06).flatten ++ ( + rURI(".").a(ldes.EventStream) + -- ldes.timestampPath ->- sosa.resultTime + -- tree.shape ->- rURI("flows-shacl") + -- tree.view ->- rURI("") + -- tree.member ->- rURI("#4493") + -- tree.member ->- rURI("#48") + -- tree.member ->- rURI("#3003") ).graph.triples.toSeq - case D09_07 => - (rURI("").a(tree.Node) - -- tree.relation ->- ( - BNode().a(tree.LessThanRelation) - -- tree.node ->- rURI("2021-09-07") - -- tree.path ->- sosa.resultTime - -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) - )).graph ++ obsrvs(D09_07).flatten ++ ( - rURI(".").a(ldes.EventStream) - -- ldes.timestampPath ->- sosa.resultTime - -- tree.shape ->- rURI("flows-shacl") - -- tree.view ->- rURI("") - -- tree.member ->- rURI("#658") - -- tree.member ->- rURI("#3074") - -- tree.member ->- rURI("#637") + case D09_07 => + (rURI("").a(tree.Node) + -- tree.relation ->- ( + BNode().a(tree.LessThanRelation) + -- tree.node ->- rURI("2021-09-06") + -- tree.path ->- sosa.resultTime + -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) + )).graph ++ obsrvs(D09_07).flatten ++ ( + rURI(".").a(ldes.EventStream) + -- ldes.timestampPath ->- sosa.resultTime + -- tree.shape ->- rURI("flows-shacl") + -- tree.view ->- rURI("") + -- tree.member ->- rURI("#658") + -- tree.member ->- rURI("#3074") + -- tree.member ->- rURI("#637") ).graph.triples.toSeq - IO(res.resolveAgainst(AbsoluteUrl.parse(url.value))) + From 36018382150b9e94401ecf390e4e4929f7b06436 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Sat, 11 Mar 2023 15:02:58 +0100 Subject: [PATCH 04/42] Add test with links pointing nowhere --- .../test/scala/run/cosy/ld/JenaWebTest.scala | 2 +- .../scala/run/cosy/ld/LdesBrokenWebTest.scala | 90 ++++++++++++++++++ .../run/cosy/ld/LdesCircularWebTest.scala | 93 ------------------- ...iLdesWWW.scala => BrokenMiniLdesWWW.scala} | 21 ++++- .../scala/run/cosy/ld/ldes/MiniLdesWWW.scala | 85 +++++++++-------- 5 files changed, 152 insertions(+), 139 deletions(-) create mode 100644 ldes/shared/src/test/scala/run/cosy/ld/LdesBrokenWebTest.scala delete mode 100644 ldes/shared/src/test/scala/run/cosy/ld/LdesCircularWebTest.scala rename ldes/shared/src/test/scala/run/cosy/ld/ldes/{CircularMiniLdesWWW.scala => BrokenMiniLdesWWW.scala} (58%) diff --git a/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala b/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala index 6cfea06..8fd8b3a 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala @@ -7,6 +7,6 @@ class JenaFoafWebTest extends FoafWebTest[R]() class JenaLdesSimpleWebTest extends LdesSimpleWebTest[R]() -class JenaLdesCircularWebTest extends LdesCircularWebTest[R]() +class JenaLdesBrokenWebTest extends LdesBrokenWebTest[R]() diff --git a/ldes/shared/src/test/scala/run/cosy/ld/LdesBrokenWebTest.scala b/ldes/shared/src/test/scala/run/cosy/ld/LdesBrokenWebTest.scala new file mode 100644 index 0000000..d2421d8 --- /dev/null +++ b/ldes/shared/src/test/scala/run/cosy/ld/LdesBrokenWebTest.scala @@ -0,0 +1,90 @@ +package run.cosy.ld + +import cats.effect.IO +import cats.effect.kernel.{Concurrent, Ref} +import io.lemonlabs as ll +import munit.CatsEffectSuite +import org.w3.banana.{Ops, RDF} +import run.cosy.ld.ldes.{BrokenMiniLdesWWW, MiniLdesWWW} + +import scala.collection.immutable.{Seq, Set} + +trait LdesBrokenWebTest[R <: RDF]()(using ops: Ops[R]) extends CatsEffectSuite: + import MiniLdesWWW.* + val miniWeb = new BrokenMiniLdesWWW[R] + given www: Web[IO, R] = miniWeb + import miniWeb.{sosa, tree, wgs84} + import ops.{*, given} + import run.cosy.ld.PNGraph.* + + test("get circular data with links going nowhere without breaking") { + import cats.syntax.traverse.{*, given} + val z: IO[fs2.Stream[IO, RDF.Graph[R]]] = + for + visitedRef <- Ref.of[IO, Set[RDF.URI[R]]](Set()) + views <- www.getPNG(URI(Collection)).map(_.rel(tree.view)) + yield fs2.Stream.unfoldLoopEval(views) { views => + import cats.syntax.traverse.{*, given} + for + v <- visitedRef.get + // here we make sure we don't visit the same page twice and we don't fail on missing pages + pagesEither <- views + .collect { + case ung: UriNGraph[R] if !v.contains(ung.point.fragmentLess) => + ung.jump[IO].attempt + } + .sequence + pages = pagesEither.collect{ case Right(png) => png } + _ <- visitedRef.update { v => + val urls = pages.map(_.name) + v.union(urls.toSet) + } + yield + val nextPages: Seq[UriNGraph[R]] = pages + .rel(tree.relation) + .filterType(tree.GreaterThanRelation) + .rel(tree.node) + .collect { case ung: UriNGraph[R] => ung } + // we need to place the pointer on the Collection of each page + val collInPages: Seq[UriNGraph[R]] = + val uc = URI(Collection) + pages.map(ung => new UriNGraph[R](uc, ung.name, ung.graph)) + val obs: RDF.Graph[R] = collInPages + .rel(tree.member) + .map(png => + png.collect( + rdf.typ, + wgs84.location, + sosa.hasSimpleResult, + sosa.madeBySensor, + sosa.observedProperty, + sosa.resultTime + )() + ) + .fold(Graph.empty)((g1, g2) => g1 union g2) + ( + obs, + if nextPages.isEmpty then None + else Some(nextPages) + ) + } + + z.flatMap { graphs => + graphs.compile.toList.map { lst => + val g: RDF.Graph[R] = lst.fold(Graph.empty)((g1, g2) => g1 union g2) + val gTrpls: Set[RDF.Triple[R]] = Set(g.triples.toSeq*) + val expectedTrpls: Seq[RDF.Triple[R]] = miniWeb.obsrvs.iterator + .map { (uristr, rtriples) => + val u = ll.uri.AbsoluteUrl.parse(uristr) + rtriples.flatten.map((rt: RDF.rTriple[R]) => rt.resolveAgainst(u)._1) + } + .toSeq + .flatten + val expected: Set[RDF.Triple[R]] = Set(expectedTrpls*) + // we can compare them as sets as we have no blank nodes + assertEquals(expected.diff(gTrpls), Set.empty) + assertEquals(gTrpls.size, expected.size) + assertEquals(gTrpls, expected) + } + } + } diff --git a/ldes/shared/src/test/scala/run/cosy/ld/LdesCircularWebTest.scala b/ldes/shared/src/test/scala/run/cosy/ld/LdesCircularWebTest.scala deleted file mode 100644 index 0d87720..0000000 --- a/ldes/shared/src/test/scala/run/cosy/ld/LdesCircularWebTest.scala +++ /dev/null @@ -1,93 +0,0 @@ -package run.cosy.ld - -import cats.effect.IO -import cats.effect.kernel.{Concurrent, Ref} -import io.lemonlabs as ll -import munit.CatsEffectSuite -import org.w3.banana.{Ops, RDF} -import run.cosy.ld.ldes.{CircularMiniLdesWWW, MiniLdesWWW} - -import scala.collection.immutable.{Seq, Set} - -trait LdesCircularWebTest[R <: RDF]()(using ops: Ops[R]) extends CatsEffectSuite: - import MiniLdesWWW.* - val miniWeb = new CircularMiniLdesWWW[R] - given www: Web[IO, R] = miniWeb - import miniWeb.{sosa, tree, wgs84} - import ops.{*, given} - import run.cosy.ld.PNGraph.* - - test("get circular data but recognise circularity") { - import cats.syntax.traverse.{*, given} - val z: IO[fs2.Stream[IO, RDF.Graph[R]]] = { - for { - visitedRef <- Ref.of[IO, Set[RDF.URI[R]]](Set()) - views <- www.getPNG(URI(Collection)).map(_.rel(tree.view)) - } yield { - fs2.Stream - .unfoldLoopEval(views) { views => - import cats.syntax.traverse.{*, given} - for - v <- visitedRef.get - // here we make sure we don't visit the same page twice. - pages <- views.collect { - case ung: UriNGraph[R] if !v.contains(ung.point.fragmentLess) => ung.jump[IO] - }.sequence - _ <- visitedRef.update { v => - val urls = pages.map(_.name) - v.union(urls.toSet) - } - yield { - val nextPages: Seq[UriNGraph[R]] = pages - .asInstanceOf[Seq[UriNGraph[R]]] - .rel(tree.relation) - .filterType(tree.GreaterThanRelation) - .rel(tree.node) - .collect { case ung: UriNGraph[R] => ung } - // we need to place the pointer on the Collection of each page - val collInPages: Seq[UriNGraph[R]] = - val uc = URI(Collection) - pages.map(ung => new UriNGraph[R](uc, ung.name, ung.graph)) - val obs: RDF.Graph[R] = collInPages - .rel(tree.member) - .map(png => - png.collect( - rdf.typ, - wgs84.location, - sosa.hasSimpleResult, - sosa.madeBySensor, - sosa.observedProperty, - sosa.resultTime - )() - ) - .fold(Graph.empty)((g1, g2) => g1 union g2) - ( - obs, - if nextPages.isEmpty then None - else Some(nextPages) - ) - } - - } - } - } - - z.flatMap { graphs => - graphs.compile.toList.map { lst => - val g: RDF.Graph[R] = lst.fold(Graph.empty)((g1, g2) => g1 union g2) - val gTrpls: Set[RDF.Triple[R]] = Set(g.triples.toSeq*) - val expectedTrpls: Seq[RDF.Triple[R]] = miniWeb.obsrvs.iterator - .map { (uristr, rtriples) => - val u = ll.uri.AbsoluteUrl.parse(uristr) - rtriples.flatten.map((rt: RDF.rTriple[R]) => rt.resolveAgainst(u)._1) - } - .toSeq - .flatten - val expected: Set[RDF.Triple[R]] = Set(expectedTrpls*) - // we can compare them as sets as we have no blank nodes - assertEquals(expected.diff(gTrpls), Set.empty) - assertEquals(gTrpls.size, expected.size) - assertEquals(gTrpls, expected) - } - } - } diff --git a/ldes/shared/src/test/scala/run/cosy/ld/ldes/CircularMiniLdesWWW.scala b/ldes/shared/src/test/scala/run/cosy/ld/ldes/BrokenMiniLdesWWW.scala similarity index 58% rename from ldes/shared/src/test/scala/run/cosy/ld/ldes/CircularMiniLdesWWW.scala rename to ldes/shared/src/test/scala/run/cosy/ld/ldes/BrokenMiniLdesWWW.scala index fbda701..c4e7e6f 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/ldes/CircularMiniLdesWWW.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/ldes/BrokenMiniLdesWWW.scala @@ -4,16 +4,17 @@ import cats.effect.IO import org.w3.banana.* import org.w3.banana.diesel.{*, given} -class CircularMiniLdesWWW[R <: RDF](using ops: Ops[R]) extends MiniLdesWWW[R]: +/** Ldes example with circularity and link pointing nowhere */ +class BrokenMiniLdesWWW[R <: RDF](using ops: Ops[R]) extends MiniLdesWWW[R]: import MiniLdesWWW.* import ops.{*, given} // we introduce a circularity in the pagination of D09_07 - override def getRelativeGraph(url: RDF.URI[R]): RDF.rGraph[R] = + override def getRelativeGraph(url: RDF.URI[R]): Option[RDF.rGraph[R]] = import scala.language.implicitConversions url.value match case D09_07 => - (rURI("").a(tree.Node) + val g: RDF.rGraph[R] = (rURI("").a(tree.Node) -- tree.relation ->- ( BNode().a(tree.LessThanRelation) -- tree.node ->- rURI("2021-09-07") @@ -22,7 +23,13 @@ class CircularMiniLdesWWW[R <: RDF](using ops: Ops[R]) extends MiniLdesWWW[R]: ) -- tree.relation ->- ( BNode().a(tree.GreaterThanRelation) - -- tree.node ->- rURI("2021-09-05") //warning: error circularity + -- tree.node ->- rURI("2021-09-09") // warning: does not exist + -- tree.path ->- sosa.resultTime + -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) + ) + -- tree.relation ->- ( + BNode().a(tree.GreaterThanRelation) + -- tree.node ->- rURI("2021-09-05") // warning: error circularity -- tree.path ->- sosa.resultTime -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) )).graph ++ obsrvs(D09_07).flatten ++ ( @@ -34,4 +41,10 @@ class CircularMiniLdesWWW[R <: RDF](using ops: Ops[R]) extends MiniLdesWWW[R]: -- tree.member ->- rURI("#3074") -- tree.member ->- rURI("#637") ).graph.triples.toSeq + Some(g) + case Collection => + // we add a link to a non existing view + super.getRelativeGraph(url).map{rg => + rg + rTriple(rURI(""), tree.view, rURI("2021-09-20")) + } case _ => super.getRelativeGraph(url) diff --git a/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala b/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala index c1e0a2a..fd5a4a9 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala @@ -6,6 +6,7 @@ import org.w3.banana.diesel.{*, given} import org.w3.banana.{diesel, *} import run.cosy.ld.* import run.cosy.ld.ldes.prefix as ldesPre + import scala.language.implicitConversions object MiniLdesWWW: @@ -31,8 +32,9 @@ class MiniLdesWWW[R <: RDF](using ops: Ops[R]) extends Web[IO, R]: def cropProp(prop: String) = URI("https://data.cropland.be/measure#" + prop) def get(url: RDF.URI[R]): IO[RDF.Graph[R]] = - val res: RDF.rGraph[R] = getRelativeGraph(url) - IO(res.resolveAgainst(AbsoluteUrl.parse(url.value))) + getRelativeGraph(url) match + case Some(res) => IO(res.resolveAgainst(AbsoluteUrl.parse(url.value))) + case None => IO.raiseError[RDF.Graph[R]](new Exception(s"resource $url not reachable")) def getPNG(url: RDF.URI[R]): IO[UriNGraph[R]] = val doc = url.fragmentLess @@ -133,22 +135,24 @@ class MiniLdesWWW[R <: RDF](using ops: Ops[R]) extends Web[IO, R]: ) ) ) - - def getRelativeGraph(url: RDF.URI[R]): RDF.rGraph[R] = - url.value match - case Collection => - (rURI("").a(ldes.EventStream) - -- ldes.timestampPath ->- sosa.resultTime - -- tree.shape ->- rURI("flows-shacl") - -- tree.view ->- rURI("2021-09-05")).graph - case D09_05 => - (rURI("") -- rdf.typ ->- tree.Node - -- tree.relation ->- ( + + def getRelativeGraph(url: RDF.URI[R]): Option[RDF.rGraph[R]] = + relG.lift(url.value) + + def relG: PartialFunction[String, RDF.rGraph[R]] = + case Collection => + (rURI("").a(ldes.EventStream) + -- ldes.timestampPath ->- sosa.resultTime + -- tree.shape ->- rURI("flows-shacl") + -- tree.view ->- rURI("2021-09-05")).graph + case D09_05 => + (rURI("") -- rdf.typ ->- tree.Node + -- tree.relation ->- ( BNode() -- rdf.typ ->- tree.GreaterThanRelation -- tree.node ->- rURI("2021-09-06") -- tree.path ->- sosa.resultTime -- tree.value ->- ("2021-09-06T00:00:00+02" ^^ xsd.dateTimeStamp) - )).graph ++ obsrvs(D09_05).flatten ++ ( + )).graph ++ obsrvs(D09_05).flatten ++ ( rURI(".").a(ldes.EventStream) -- ldes.timestampPath ->- sosa.resultTime -- tree.shape ->- rURI("flows-shacl") @@ -157,42 +161,41 @@ class MiniLdesWWW[R <: RDF](using ops: Ops[R]) extends Web[IO, R]: -- tree.member ->- rURI("#482") -- tree.member ->- rURI("#4464") ).graph.triples.toSeq - case D09_06 => - (rURI("").a(tree.Node) - -- tree.relation ->- ( + case D09_06 => + (rURI("").a(tree.Node) + -- tree.relation ->- ( BNode().a(tree.LessThanRelation) -- tree.node ->- rURI("2021-09-05") -- tree.path ->- sosa.resultTime -- tree.value ->- ("2021-09-06T00:00:00+02" ^^ xsd.dateTimeStamp) - ) - -- tree.relation ->- ( + ) + -- tree.relation ->- ( BNode().a(tree.GreaterThanRelation) -- tree.node ->- rURI("2021-09-07") -- tree.path ->- sosa.resultTime -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) - )).graph ++ obsrvs(D09_06).flatten ++ ( - rURI(".").a(ldes.EventStream) - -- ldes.timestampPath ->- sosa.resultTime - -- tree.shape ->- rURI("flows-shacl") - -- tree.view ->- rURI("") - -- tree.member ->- rURI("#4493") - -- tree.member ->- rURI("#48") - -- tree.member ->- rURI("#3003") - ).graph.triples.toSeq - case D09_07 => - (rURI("").a(tree.Node) - -- tree.relation ->- ( + )).graph ++ obsrvs(D09_06).flatten ++ ( + rURI(".").a(ldes.EventStream) + -- ldes.timestampPath ->- sosa.resultTime + -- tree.shape ->- rURI("flows-shacl") + -- tree.view ->- rURI("") + -- tree.member ->- rURI("#4493") + -- tree.member ->- rURI("#48") + -- tree.member ->- rURI("#3003") + ).graph.triples.toSeq + case D09_07 => + (rURI("").a(tree.Node) + -- tree.relation ->- ( BNode().a(tree.LessThanRelation) -- tree.node ->- rURI("2021-09-06") -- tree.path ->- sosa.resultTime -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) - )).graph ++ obsrvs(D09_07).flatten ++ ( - rURI(".").a(ldes.EventStream) - -- ldes.timestampPath ->- sosa.resultTime - -- tree.shape ->- rURI("flows-shacl") - -- tree.view ->- rURI("") - -- tree.member ->- rURI("#658") - -- tree.member ->- rURI("#3074") - -- tree.member ->- rURI("#637") - ).graph.triples.toSeq - + )).graph ++ obsrvs(D09_07).flatten ++ ( + rURI(".").a(ldes.EventStream) + -- ldes.timestampPath ->- sosa.resultTime + -- tree.shape ->- rURI("flows-shacl") + -- tree.view ->- rURI("") + -- tree.member ->- rURI("#658") + -- tree.member ->- rURI("#3074") + -- tree.member ->- rURI("#637") + ).graph.triples.toSeq From d43790d58f215ef700be176a0b2e34e46fbc1b9c Mon Sep 17 00:00:00 2001 From: Henry Story Date: Mon, 13 Mar 2023 21:57:43 +0100 Subject: [PATCH 05/42] Web implementation using http4s --- build.sbt | 4 +- .../src/main/scala/run/cosy/ld/PNGraph.scala | 6 +-- .../main/scala/run/cosy/ld/http4s/H4Web.scala | 45 +++++++++++++++++++ .../run/cosy/ld/http4s}/RDFDecoders.scala | 4 +- .../scala/run/cosy}/web/util/UrlUtil.scala | 2 +- project/Dependencies.scala | 4 +- 6 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 ldes/shared/src/main/scala/run/cosy/ld/http4s/H4Web.scala rename {wallet/shared/src/main/scala/org/w3/banana/http4sIO => ldes/shared/src/main/scala/run/cosy/ld/http4s}/RDFDecoders.scala (98%) rename {wallet/shared/src/main/scala/net/bblfish => ldes/shared/src/main/scala/run/cosy}/web/util/UrlUtil.scala (98%) diff --git a/build.sbt b/build.sbt index bcc0dbc..6020b3d 100644 --- a/build.sbt +++ b/build.sbt @@ -113,8 +113,10 @@ lazy val ldes = crossProject(JVMPlatform) resolvers += sonatypeSNAPSHOT, libraryDependencies ++= Seq( banana.bananaRdf.value, + banana.bananaIO.value, // cats.effect.value, - cats.fs2.value + cats.fs2.value, + http4s.client.value ), libraryDependencies ++= Seq( munit.value % Test, diff --git a/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala b/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala index 52e22ba..75f3dee 100644 --- a/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala +++ b/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala @@ -9,10 +9,10 @@ import org.w3.banana.{Ops, RDF} trait Web[F[_]: Concurrent, R <: RDF]: /** get a Graph for the given URL (without a fragment) */ def get(url: RDF.URI[R]): F[RDF.Graph[R]] - - /** get Pointed Named Graph for given url */ + +/** get Pointed Named Graph for given url */ def getPNG(url: RDF.URI[R]): F[UriNGraph[R]] - + /** A Pointed Named Graph, ie, a pointer into a NamedGraph We don't use a case class here as * equality between PNGgraphs is complicated by the need to prove isomorphism between graphs, and * the nodes have to be equivalent. diff --git a/ldes/shared/src/main/scala/run/cosy/ld/http4s/H4Web.scala b/ldes/shared/src/main/scala/run/cosy/ld/http4s/H4Web.scala new file mode 100644 index 0000000..ddde30d --- /dev/null +++ b/ldes/shared/src/main/scala/run/cosy/ld/http4s/H4Web.scala @@ -0,0 +1,45 @@ +package run.cosy.ld.http4s + +import cats.effect.Concurrent +import io.lemonlabs.uri.Url +import org.http4s as h4s +import org.http4s.client.Client +import org.w3.banana.RDF +import org.w3.banana.RDF.URI +import run.cosy.ld.http4s.RDFDecoders +import run.cosy.ld.{UriNGraph, Web} +import run.cosy.web.util.UrlUtil.http4sUrlToLLUrl + +/** + * Web implementation in http4s + * todo: we need a client that can find out about redirects, so that we can + * correctly name the resource + * todo: we also need a web cache that can be queried and updated (or perhaps that would use + * this) + */ +class H4Web[F[_]: Concurrent, R <: RDF]( + client: Client[F] +)(using + val rdfDecoders: RDFDecoders[F, R] +) extends Web[F, R]: + import cats.syntax.all.* + import rdfDecoders.ops + import rdfDecoders.allrdf + import ops.{*, given} + + override def get(url: RDF.URI[R]): F[RDF.Graph[R]] = + for + u <- Concurrent[F].fromEither(h4s.Uri.fromString(url.value)) + doc = u.withoutFragment + rG <- client.fetchAs[RDF.rGraph[R]]( + h4s.Request( + uri = doc, + headers = h4s.Headers(rdfDecoders.allRdfAccept) + ) + ) + yield rG.resolveAgainst(http4sUrlToLLUrl(doc).toAbsoluteUrl) + + //todo: this should really use client.fetchPNG + override def getPNG(url: RDF.URI[R]): F[UriNGraph[R]] = + val doc = url.fragmentLess + get(doc).map(g => new UriNGraph(url, doc, g)) diff --git a/wallet/shared/src/main/scala/org/w3/banana/http4sIO/RDFDecoders.scala b/ldes/shared/src/main/scala/run/cosy/ld/http4s/RDFDecoders.scala similarity index 98% rename from wallet/shared/src/main/scala/org/w3/banana/http4sIO/RDFDecoders.scala rename to ldes/shared/src/main/scala/run/cosy/ld/http4s/RDFDecoders.scala index ad43d91..144bfde 100644 --- a/wallet/shared/src/main/scala/org/w3/banana/http4sIO/RDFDecoders.scala +++ b/ldes/shared/src/main/scala/run/cosy/ld/http4s/RDFDecoders.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.w3.banana.http4sIO +package run.cosy.ld.http4s import cats.{Applicative, FlatMap} import cats.data.EitherT @@ -41,7 +41,7 @@ import org.http4s.headers.Accept import org.w3.banana.RDF.rGraph class RDFDecoders[F[_], Rdf <: RDF](using - ops: Ops[Rdf], + val ops: Ops[Rdf], cc: Concurrent[F], turtleReader: RelRDFReader[Rdf, Try, Turtle], rdfXmlReader: RelRDFReader[Rdf, Try, RDFXML], diff --git a/wallet/shared/src/main/scala/net/bblfish/web/util/UrlUtil.scala b/ldes/shared/src/main/scala/run/cosy/web/util/UrlUtil.scala similarity index 98% rename from wallet/shared/src/main/scala/net/bblfish/web/util/UrlUtil.scala rename to ldes/shared/src/main/scala/run/cosy/web/util/UrlUtil.scala index 22ab787..25a81b6 100644 --- a/wallet/shared/src/main/scala/net/bblfish/web/util/UrlUtil.scala +++ b/ldes/shared/src/main/scala/run/cosy/web/util/UrlUtil.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package net.bblfish.web.util +package run.cosy.web.util import com.comcast.ip4s import io.lemonlabs.uri as ll diff --git a/project/Dependencies.scala b/project/Dependencies.scala index b99c04e..da06ccd 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -37,8 +37,9 @@ object Dependencies { lazy val fs2 = Def.setting("co.fs2" %% "fs2-core" % "3.6.1") // https://github.com/typelevel/munit-cats-effect + // https://search.maven.org/artifact/org.typelevel/munit-cats-effect_3/2.0.0-M3/jar lazy val munitEffect = - Def.setting("org.typelevel" %%% "munit-cats-effect-3" % "1.0.7") + Def.setting("org.typelevel" %%% "munit-cats-effect" % "2.0.0-M3") } object crypto { @@ -57,6 +58,7 @@ object Dependencies { object banana { lazy val bananaRdf = Def.setting("net.bblfish.rdf" %%% "banana-rdf" % Ver.banana) + lazy val bananaIO = Def.setting("net.bblfish.rdf" %%% "banana-jena-io-sync" % Ver.banana) lazy val bananaJena = Def.setting("net.bblfish.rdf" %%% "banana-jena-io-sync" % Ver.banana) } From 4827974fcfe986b6313aeb719ed0367af342ffb3 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Wed, 15 Mar 2023 16:07:48 +0100 Subject: [PATCH 06/42] added script to fetch ldes from reactive solid --- build.sbt | 3 +- .../src/main/scala/run/cosy/ld/PNGraph.scala | 2 +- .../main/scala/run/cosy/ldes/LdesSpider.scala | 93 +++++++++++++++++++ .../scala/run/cosy}/ldes/prefix/LDES.scala | 4 +- .../scala/run/cosy}/ldes/prefix/SOSA.scala | 2 +- .../scala/run/cosy}/ldes/prefix/TREE.scala | 2 +- .../scala/run/cosy}/ldes/prefix/WGS84.scala | 5 +- .../scala/run/cosy/ld/ldes/MiniLdesWWW.scala | 9 +- scripts/jvm/src/main/scala/scripts/Fetch.sc | 52 ++++++----- .../jvm/src/main/scala/scripts/MiniCF.scala | 68 ++++++++++++++ 10 files changed, 202 insertions(+), 38 deletions(-) create mode 100644 ldes/shared/src/main/scala/run/cosy/ldes/LdesSpider.scala rename ldes/shared/src/{test/scala/run/cosy/ld => main/scala/run/cosy}/ldes/prefix/LDES.scala (90%) rename ldes/shared/src/{test/scala/run/cosy/ld => main/scala/run/cosy}/ldes/prefix/SOSA.scala (98%) rename ldes/shared/src/{test/scala/run/cosy/ld => main/scala/run/cosy}/ldes/prefix/TREE.scala (97%) rename ldes/shared/src/{test/scala/run/cosy/ld => main/scala/run/cosy}/ldes/prefix/WGS84.scala (81%) create mode 100644 scripts/jvm/src/main/scala/scripts/MiniCF.scala diff --git a/build.sbt b/build.sbt index 6020b3d..36257da 100644 --- a/build.sbt +++ b/build.sbt @@ -173,12 +173,13 @@ lazy val scripts = crossProject(JVMPlatform) // crypto.bobcats.value classifier ("tests-sources") // bobcats test examples soources, // ) // ) - .dependsOn(authN) + .dependsOn(/*authN,*/ ldes) .jvmSettings( libraryDependencies ++= Seq( crypto.bobcats.value classifier ("tests"), // bobcats test examples, crypto.bobcats.value classifier ("tests-sources"), // bobcats test examples soources, other.scalaUri.value, + http4s.ember_client.value, crypto.nimbusJWT_JDK.value, crypto.bouncyJCA_JDK.value ) diff --git a/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala b/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala index 75f3dee..9ab19f1 100644 --- a/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala +++ b/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala @@ -87,7 +87,7 @@ trait SubjPNGraph[R <: RDF] extends PNGraph[R]: end SubjPNGraph /** A PNG where the point is a URI */ -class UriNGraph[R <: RDF]( +case class UriNGraph[R <: RDF]( val point: URI[R], val name: URI[R], val graph: Graph[R] diff --git a/ldes/shared/src/main/scala/run/cosy/ldes/LdesSpider.scala b/ldes/shared/src/main/scala/run/cosy/ldes/LdesSpider.scala new file mode 100644 index 0000000..a2a587a --- /dev/null +++ b/ldes/shared/src/main/scala/run/cosy/ldes/LdesSpider.scala @@ -0,0 +1,93 @@ +package run.cosy.ldes + +import cats.effect.{IO, Ref} +import cats.effect.kernel.Concurrent +import org.w3.banana.RDF.{Graph, Node, Statement, URI} +import org.w3.banana.{Ops, RDF} +import run.cosy.ld.{PNGraph, UriNGraph, Web} + +import scala.collection.immutable.Set + +class LdesSpider[F[_]: Concurrent, R <: RDF]( + using www: Web[F, R], + ops: Ops[R] +): + + import ops.{*, given} + import org.w3.banana.prefix + import run.cosy.ldes.prefix as ldesPre + + val foaf = prefix.FOAF[R] + val tree = ldesPre.TREE[R] + val sosa = ldesPre.SOSA[R] + val wgs84 = ldesPre.WGS84[R] + val ldes = ldesPre.LDES[R] + + + /** given the ldes stream URL, crawl the nodes of that stream */ + def crawl(stream: RDF.URI[R]): F[fs2.Stream[F, RDF.Graph[R]]] = + import cats.syntax.all.toFunctorOps + import cats.syntax.flatMap.toFlatMapOps + for + visitedRef <- Ref.of[F, Set[RDF.URI[R]]](Set()) + views <- www.getPNG(stream).map(_.rel(tree.view)) + yield crawlNodesForward(stream, views, visitedRef) + + /* This crawls pages forward and collects all tree.member observations... + * This is very much tied to a particular use of ldes + * todo: it should be able to fetch the shEx to find out what to + * collect, or an pattern object should be passed. + * @returns an fs2.Stream of such observation mini graphs + **/ + def crawlNodesForward( + stream: RDF.URI[R], + startNodes: Seq[PNGraph[R]], + visitedRef: Ref[F, Set[RDF.URI[R]]] + ): fs2.Stream[F, RDF.Graph[R]] = + import cats.syntax.all.toFlatMapOps + fs2.Stream.unfoldLoopEval(startNodes) { nodes => + import cats.syntax.traverse.{*, given} + import cats.syntax.all.toFunctorOps + for + v <- visitedRef.get + // here we make sure we don't visit the same page twice and we don't fail on missing pages + pagesEither <- nodes.collect { + case ung: UriNGraph[R] if !v.contains(ung.point.fragmentLess) => + import cats.implicits.catsSyntaxApplicativeError + ung.jump[F].attempt + }.sequence + pages: Seq[UriNGraph[R]] = pagesEither.collect { case Right(png) => png } + _ <- visitedRef.update { v => + val urls = pages.map(_.name) + v.union(urls.toSet) + } + yield + val nextPages: Seq[UriNGraph[R]] = pages + .rel(tree.relation) + .filterType(tree.GreaterThanRelation) + .rel(tree.node) + .collect { case ung: UriNGraph[R] => ung } + // we need to place the pointer on the Collection of each page + val collInPages: Seq[UriNGraph[R]] = + pages.map(ung => new UriNGraph[R](stream, ung.name, ung.graph)) + val obs: RDF.Graph[R] = collInPages + .rel(tree.member) + .map(png => + png.collect( + rdf.typ, + wgs84.location, + sosa.hasSimpleResult, + sosa.madeBySensor, + sosa.observedProperty, + sosa.resultTime + )() + ) + .fold(Graph.empty)((g1, g2) => g1 union g2) + ( + obs, + if nextPages.isEmpty then None + else Some(nextPages) + ) + } + end crawlNodesForward + diff --git a/ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/LDES.scala b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/LDES.scala similarity index 90% rename from ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/LDES.scala rename to ldes/shared/src/main/scala/run/cosy/ldes/prefix/LDES.scala index 342842e..54bac8f 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/LDES.scala +++ b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/LDES.scala @@ -1,6 +1,6 @@ -package run.cosy.ld.ldes.prefix +package run.cosy.ldes.prefix -import org.w3.banana.{Ops, RDF, PrefixBuilder} +import org.w3.banana.{Ops, PrefixBuilder, RDF} object LDES: def apply[Rdf <: RDF](using ops: Ops[Rdf]) = new LDES() diff --git a/ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/SOSA.scala b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/SOSA.scala similarity index 98% rename from ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/SOSA.scala rename to ldes/shared/src/main/scala/run/cosy/ldes/prefix/SOSA.scala index aee894f..b38f62a 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/SOSA.scala +++ b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/SOSA.scala @@ -1,4 +1,4 @@ -package run.cosy.ld.ldes.prefix +package run.cosy.ldes.prefix import org.w3.banana.{Ops, PrefixBuilder, RDF} diff --git a/ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/TREE.scala b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/TREE.scala similarity index 97% rename from ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/TREE.scala rename to ldes/shared/src/main/scala/run/cosy/ldes/prefix/TREE.scala index 60cfb85..786c272 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/TREE.scala +++ b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/TREE.scala @@ -1,4 +1,4 @@ -package run.cosy.ld.ldes.prefix +package run.cosy.ldes.prefix import org.w3.banana.{Ops, PrefixBuilder, RDF} diff --git a/ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/WGS84.scala b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/WGS84.scala similarity index 81% rename from ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/WGS84.scala rename to ldes/shared/src/main/scala/run/cosy/ldes/prefix/WGS84.scala index 2693143..d7553a2 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/ldes/prefix/WGS84.scala +++ b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/WGS84.scala @@ -1,7 +1,6 @@ -package run.cosy.ld.ldes.prefix +package run.cosy.ldes.prefix -import org.w3.banana.PrefixBuilder -import org.w3.banana.{RDF,Ops} +import org.w3.banana.{Ops, PrefixBuilder, RDF} object WGS84: def apply[R <: RDF](using Ops[R]) = new WGS84() diff --git a/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala b/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala index fd5a4a9..d90c1b5 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala @@ -6,6 +6,7 @@ import org.w3.banana.diesel.{*, given} import org.w3.banana.{diesel, *} import run.cosy.ld.* import run.cosy.ld.ldes.prefix as ldesPre +import run.cosy.ldes.prefix.{LDES, SOSA, TREE, WGS84} import scala.language.implicitConversions @@ -20,10 +21,10 @@ class MiniLdesWWW[R <: RDF](using ops: Ops[R]) extends Web[IO, R]: import MiniLdesWWW.* import ops.{*, given} val foaf = prefix.FOAF[R] - val tree = ldesPre.TREE[R] - val sosa = ldesPre.SOSA[R] - val wgs84 = ldesPre.WGS84[R] - val ldes = ldesPre.LDES[R] + val tree = TREE[R] + val sosa = SOSA[R] + val wgs84 = WGS84[R] + val ldes = LDES[R] def area(loc: String) = rURI("area#d" + loc) def pzDev(num: String) = URI("https://data.politie.be/sensor/dev#" + num) diff --git a/scripts/jvm/src/main/scala/scripts/Fetch.sc b/scripts/jvm/src/main/scala/scripts/Fetch.sc index 2a6148c..f4686ca 100644 --- a/scripts/jvm/src/main/scala/scripts/Fetch.sc +++ b/scripts/jvm/src/main/scala/scripts/Fetch.sc @@ -1,22 +1,20 @@ - -import bobcats.{AsymmetricKeyAlg, PKCS8KeySpec, Signer, Verifier} -import bobcats.util.BouncyJavaPEMUtils.getPrivateKeySpec -import net.bblfish.wallet.{BasicId, BasicWallet} import _root_.io.lemonlabs.uri as ll -import cats.effect.IO +import bobcats.util.BouncyJavaPEMUtils.getPrivateKeySpec +import bobcats.{AsymmetricKeyAlg, PKCS8KeySpec, Signer, Verifier} import cats.effect.* +import cats.effect.unsafe.IORuntime +import net.bblfish.app.auth.AuthNClient +import net.bblfish.wallet.{BasicId, BasicWallet} import org.w3.banana.jena.JenaRdf import org.w3.banana.jena.JenaRdf.ops import ops.given -import org.w3.banana.jena.JenaRdf.R -import scodec.bits.ByteVector -import cats.effect.unsafe.IORuntime -import net.bblfish.app.auth.AuthNClient import org.http4s.Uri as H4Uri import org.w3.banana.http4sIO.RDFDecoders +import org.w3.banana.jena.JenaRdf.R +import scodec.bits.ByteVector implicit val runtime: IORuntime = cats.effect.unsafe.IORuntime.global -import org.http4s.ember.client._ -import org.http4s.client._ +import org.http4s.client.* +import org.http4s.ember.client.* val priv = """-----BEGIN PRIVATE KEY----- MIIEvgIBADALBgkqhkiG9w0BAQoEggSqMIIEpgIBAAKCAQEAr4tmm3r20Wd/Pbqv @@ -47,28 +45,32 @@ val priv = """-----BEGIN PRIVATE KEY----- rOjr9w349JooGXhOxbu8nOxX -----END PRIVATE KEY-----""" -val pkcs8K: PKCS8KeySpec[AsymmetricKeyAlg] = getPrivateKeySpec(priv,AsymmetricKeyAlg.RSA_PSS_Key).get +val pkcs8K: PKCS8KeySpec[AsymmetricKeyAlg] = + getPrivateKeySpec(priv, AsymmetricKeyAlg.RSA_PSS_Key).get //val keyUrl: ll.Url = ll.Url("http://127.0.0.1:8080/rfcKey") val keyUrl = URI("http://localhost:8080/rfcKey#") -val signerF: IO[ByteVector => IO[ByteVector]] = Signer[IO].build(pkcs8K, bobcats.AsymmetricKeyAlg.`rsa-pss-sha512`) +val signerF: IO[ByteVector => IO[ByteVector]] = + Signer[IO].build(pkcs8K, bobcats.AsymmetricKeyAlg.`rsa-pss-sha512`) import org.w3.banana.jena.io.JenaRDFReader.given import org.w3.banana.jena.io.JenaRDFWriter.given -val keyid = net.bblfish.wallet.KeyData[IO,R](keyUrl,signerF) +val keyid = net.bblfish.wallet.KeyData[IO, R](keyUrl, signerF) -given dec: RDFDecoders[IO,R] = new RDFDecoders() +given dec: RDFDecoders[IO, R] = new RDFDecoders() import org.http4s.syntax.all.uri -def ioStr(uri: H4Uri) = EmberClientBuilder.default[IO].build.use { (client: Client[IO] ) => - import org.http4s.client.middleware.Logger - given loggedClient: Client[IO] = Logger[IO](true,true,logAction = Some(str => IO(System.out.println(str))))(client) - val bw = BasicWallet[IO, R]( - Map(), - Seq(keyid) - ) - val newClient: Client[IO] = AuthNClient[IO].apply(bw)(loggedClient) - newClient.expect[String](uri) -} +def ioStr(uri: H4Uri): IO[String] = + EmberClientBuilder.default[IO].build.use { (client: Client[IO]) => + import org.http4s.client.middleware.Logger + given loggedClient: Client[IO] = + Logger[IO](true, true, logAction = Some(str => IO(System.out.println(str))))(client) + val bw = BasicWallet[IO, R]( + Map(), + Seq(keyid) + ) + val newClient: Client[IO] = AuthNClient[IO].apply(bw)(loggedClient) + newClient.expect[String](uri) + } //ioStr(uri"http://localhost:8080/").unsafeRunSync() //ioStr(uri"http://localhost:8080/protected/").unsafeRunSync() ioStr(uri"http://localhost:8080/protected/README").unsafeRunSync() diff --git a/scripts/jvm/src/main/scala/scripts/MiniCF.scala b/scripts/jvm/src/main/scala/scripts/MiniCF.scala new file mode 100644 index 0000000..86ab8c2 --- /dev/null +++ b/scripts/jvm/src/main/scala/scripts/MiniCF.scala @@ -0,0 +1,68 @@ +package scripts + +// This fetches data from a mini City Flows web server +import cats.effect.unsafe.IORuntime +import cats.effect.{unsafe, Concurrent, IO, Ref} +import org.http4s.EntityDecoder +import org.http4s.client.Client +import org.http4s.ember.client.* +import org.w3.banana.* +import org.w3.banana.RDF.Graph +import run.cosy.ld.http4s.{H4Web, RDFDecoders} +import run.cosy.ld.{PNGraph, UriNGraph, Web} +import run.cosy.ldes.LdesSpider + +object MiniCF: + + type JR = org.w3.banana.jena.JenaRdf.type + import org.w3.banana.* + import org.w3.banana.io.{JsonLd, RDFXML, RelRDFReader, Turtle} + import org.w3.banana.jena.JenaRdf.ops + import ops.{*, given} + import org.w3.banana.jena.io.JenaRDFReader.given + given ior: unsafe.IORuntime = cats.effect.unsafe.IORuntime.global + + given rdfDecoders: RDFDecoders[IO, JR] = new RDFDecoders[IO, JR] + + @main + def crawlContainer(stream: String = "http://localhost:8080/ldes/miniCityFlows/stream#"): Unit = + val ioStr: IO[fs2.Stream[IO, RDF.Graph[JR]]] = + EmberClientBuilder.default[IO].build.use { (client: Client[IO]) => + given web: Web[IO, JR] = new H4Web[IO, JR](client) + val spider: LdesSpider[IO, JR] = new LdesSpider[IO, JR] + println("streamUri="+stream) + val streamUri: RDF.URI[JR] = ops.URI(stream) + spider.crawl(streamUri) + } + val l: IO[List[RDF.Graph[JR]]] = + fs2.Stream.eval(ioStr).flatten + .reduce((g1, g2) => g1.union(g2)) + .compile[IO, IO, RDF.Graph[JR]] + .toList + l.unsafeRunSync().foreach(_.triples.foreach(println)) + end crawlContainer + +// def crawlNodes(): Unit = +// import cats.syntax.all.toFlatMapOps +// val lstOfGrIO: IO[List[RDF.Graph[JR]]] = +// EmberClientBuilder.default[IO].build.use { (client: Client[IO]) => +// given web: Web[IO, JR] = new H4Web[IO, JR](client) +// val spider: LdesSpider[IO, JR] = new LdesSpider[IO, JR] +// val collectionURI = +// URI("http://localhost:8080/ldes/miniCityFlows/") +// val startNodeURI = +// URI("http://localhost:8080/ldes/miniCityFlows/2021-09-05") +// val startNodePNG = UriNGraph[JR](startNodeURI, collectionURI, Graph.empty) +// for +// ref <- Ref.of[IO, Set[RDF.URI[JR]]](Set()) +// lst <- spider +// .crawlNodesForward(collectionURI, Seq(startNodePNG), ref) +// .compile[IO, IO, RDF.Graph[JR]] +// .toList +// yield lst +// } +// val gr: Seq[RDF.Graph[JR]] = lstOfGrIO.unsafeRunSync() +// val oneGr: RDF.Graph[JR] = gr.reduce((g1, g2) => g1.union(g2)) +// ??? + +end MiniCF From f17df1fee3e65fb243dc051f296f95461dca2d7b Mon Sep 17 00:00:00 2001 From: Henry Story Date: Wed, 15 Mar 2023 17:22:38 +0100 Subject: [PATCH 07/42] stop sending Accept headers twice. --- .../main/scala/run/cosy/ld/http4s/H4Web.scala | 7 ++---- .../jvm/src/main/scala/scripts/MiniCF.scala | 23 ------------------- 2 files changed, 2 insertions(+), 28 deletions(-) diff --git a/ldes/shared/src/main/scala/run/cosy/ld/http4s/H4Web.scala b/ldes/shared/src/main/scala/run/cosy/ld/http4s/H4Web.scala index ddde30d..4f48d7f 100644 --- a/ldes/shared/src/main/scala/run/cosy/ld/http4s/H4Web.scala +++ b/ldes/shared/src/main/scala/run/cosy/ld/http4s/H4Web.scala @@ -32,14 +32,11 @@ class H4Web[F[_]: Concurrent, R <: RDF]( u <- Concurrent[F].fromEither(h4s.Uri.fromString(url.value)) doc = u.withoutFragment rG <- client.fetchAs[RDF.rGraph[R]]( - h4s.Request( - uri = doc, - headers = h4s.Headers(rdfDecoders.allRdfAccept) - ) + h4s.Request(uri = doc) ) yield rG.resolveAgainst(http4sUrlToLLUrl(doc).toAbsoluteUrl) //todo: this should really use client.fetchPNG override def getPNG(url: RDF.URI[R]): F[UriNGraph[R]] = val doc = url.fragmentLess - get(doc).map(g => new UriNGraph(url, doc, g)) + get(doc).map(g => UriNGraph(url, doc, g)) diff --git a/scripts/jvm/src/main/scala/scripts/MiniCF.scala b/scripts/jvm/src/main/scala/scripts/MiniCF.scala index 86ab8c2..ae0c733 100644 --- a/scripts/jvm/src/main/scala/scripts/MiniCF.scala +++ b/scripts/jvm/src/main/scala/scripts/MiniCF.scala @@ -30,7 +30,6 @@ object MiniCF: EmberClientBuilder.default[IO].build.use { (client: Client[IO]) => given web: Web[IO, JR] = new H4Web[IO, JR](client) val spider: LdesSpider[IO, JR] = new LdesSpider[IO, JR] - println("streamUri="+stream) val streamUri: RDF.URI[JR] = ops.URI(stream) spider.crawl(streamUri) } @@ -42,27 +41,5 @@ object MiniCF: l.unsafeRunSync().foreach(_.triples.foreach(println)) end crawlContainer -// def crawlNodes(): Unit = -// import cats.syntax.all.toFlatMapOps -// val lstOfGrIO: IO[List[RDF.Graph[JR]]] = -// EmberClientBuilder.default[IO].build.use { (client: Client[IO]) => -// given web: Web[IO, JR] = new H4Web[IO, JR](client) -// val spider: LdesSpider[IO, JR] = new LdesSpider[IO, JR] -// val collectionURI = -// URI("http://localhost:8080/ldes/miniCityFlows/") -// val startNodeURI = -// URI("http://localhost:8080/ldes/miniCityFlows/2021-09-05") -// val startNodePNG = UriNGraph[JR](startNodeURI, collectionURI, Graph.empty) -// for -// ref <- Ref.of[IO, Set[RDF.URI[JR]]](Set()) -// lst <- spider -// .crawlNodesForward(collectionURI, Seq(startNodePNG), ref) -// .compile[IO, IO, RDF.Graph[JR]] -// .toList -// yield lst -// } -// val gr: Seq[RDF.Graph[JR]] = lstOfGrIO.unsafeRunSync() -// val oneGr: RDF.Graph[JR] = gr.reduce((g1, g2) => g1.union(g2)) -// ??? end MiniCF From b6ef1a7c1cf4091833f456ae7cb78b705d2a1357 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Wed, 15 Mar 2023 22:11:41 +0100 Subject: [PATCH 08/42] fetch with wallet updated --- build.sbt | 27 +++++- .../run/cosy/ld/http4s/RDFDecoders.scala | 0 .../scala/run/cosy/web/util/UrlUtil.scala | 0 .../main/scala/scripts/AnHttpSigClient.scala | 94 +++++++++++++++++++ scripts/jvm/src/main/scala/scripts/Fetch.sc | 76 --------------- .../jvm/src/main/scala/scripts/MiniCF.scala | 6 +- .../net/bblfish/wallet/BasicAuthWallet.scala | 4 +- 7 files changed, 123 insertions(+), 84 deletions(-) rename {ldes => ioExt4s}/shared/src/main/scala/run/cosy/ld/http4s/RDFDecoders.scala (100%) rename {ldes => ioExt4s}/shared/src/main/scala/run/cosy/web/util/UrlUtil.scala (100%) create mode 100644 scripts/jvm/src/main/scala/scripts/AnHttpSigClient.scala delete mode 100644 scripts/jvm/src/main/scala/scripts/Fetch.sc diff --git a/build.sbt b/build.sbt index 36257da..78017d7 100644 --- a/build.sbt +++ b/build.sbt @@ -105,6 +105,7 @@ lazy val free = crossProject(JVMPlatform) // , JSPlatform) lazy val ldes = crossProject(JVMPlatform) .crossType(CrossType.Full) .in(file("ldes")) + .dependsOn(ioExt4s) .settings(commonSettings: _*) .settings( name := "LDES Client", @@ -112,11 +113,8 @@ lazy val ldes = crossProject(JVMPlatform) // scalacOptions := scala3jsOptions, resolvers += sonatypeSNAPSHOT, libraryDependencies ++= Seq( - banana.bananaRdf.value, - banana.bananaIO.value, // cats.effect.value, cats.fs2.value, - http4s.client.value ), libraryDependencies ++= Seq( munit.value % Test, @@ -125,11 +123,32 @@ lazy val ldes = crossProject(JVMPlatform) ) ) +// todo: should be moved closer to banana-rdf repo +lazy val ioExt4s = crossProject(JVMPlatform) + .crossType(CrossType.Full) + .in(file("ioExt4s")) + .settings(commonSettings: _*) + .settings( + name := "IO http4s ext", + description := "rdf io extensions for http4s", + // scalacOptions := scala3jsOptions, + resolvers += sonatypeSNAPSHOT, + libraryDependencies ++= Seq( + http4s.client.value, + banana.bananaIO.value, + ), + libraryDependencies ++= Seq( + munit.value % Test, + cats.munitEffect.value % Test, + ) + ) + //todo: we should split the wallet into client-wallet and the full wallet library // as clients of the wallet only need a minimal interface lazy val wallet = crossProject(JVMPlatform) // , JSPlatform) .crossType(CrossType.Full) .in(file("wallet")) + .dependsOn(ioExt4s) .settings(commonSettings: _*) .settings( name := "Solid Wallet", @@ -173,7 +192,7 @@ lazy val scripts = crossProject(JVMPlatform) // crypto.bobcats.value classifier ("tests-sources") // bobcats test examples soources, // ) // ) - .dependsOn(/*authN,*/ ldes) + .dependsOn(wallet, authN, ldes) .jvmSettings( libraryDependencies ++= Seq( crypto.bobcats.value classifier ("tests"), // bobcats test examples, diff --git a/ldes/shared/src/main/scala/run/cosy/ld/http4s/RDFDecoders.scala b/ioExt4s/shared/src/main/scala/run/cosy/ld/http4s/RDFDecoders.scala similarity index 100% rename from ldes/shared/src/main/scala/run/cosy/ld/http4s/RDFDecoders.scala rename to ioExt4s/shared/src/main/scala/run/cosy/ld/http4s/RDFDecoders.scala diff --git a/ldes/shared/src/main/scala/run/cosy/web/util/UrlUtil.scala b/ioExt4s/shared/src/main/scala/run/cosy/web/util/UrlUtil.scala similarity index 100% rename from ldes/shared/src/main/scala/run/cosy/web/util/UrlUtil.scala rename to ioExt4s/shared/src/main/scala/run/cosy/web/util/UrlUtil.scala diff --git a/scripts/jvm/src/main/scala/scripts/AnHttpSigClient.scala b/scripts/jvm/src/main/scala/scripts/AnHttpSigClient.scala new file mode 100644 index 0000000..98e81a0 --- /dev/null +++ b/scripts/jvm/src/main/scala/scripts/AnHttpSigClient.scala @@ -0,0 +1,94 @@ +package scripts + +import _root_.io.lemonlabs.uri as ll +import bobcats.util.BouncyJavaPEMUtils.getPrivateKeySpec +import bobcats.{AsymmetricKeyAlg, PKCS8KeySpec, Signer, Verifier} +import cats.effect.* +import cats.effect.unsafe.IORuntime +import net.bblfish.app.auth.AuthNClient +import net.bblfish.wallet.{BasicId, BasicWallet, KeyData} +import org.w3.banana.jena.JenaRdf +import org.w3.banana.jena.JenaRdf.ops +import ops.given +import org.http4s.Uri as H4Uri +import org.w3.banana.jena.JenaRdf.R +import run.cosy.http.headers.Rfc8941 +import run.cosy.ld.http4s.RDFDecoders +import scodec.bits.ByteVector +import org.http4s.client.* +import org.http4s.ember.client.* +import run.cosy.http.headers.SigIn.KeyId + +object AnHttpSigClient: + implicit val runtime: IORuntime = cats.effect.unsafe.IORuntime.global + + val priv = """-----BEGIN PRIVATE KEY----- + MIIEvgIBADALBgkqhkiG9w0BAQoEggSqMIIEpgIBAAKCAQEAr4tmm3r20Wd/Pbqv + P1s2+QEtvpuRaV8Yq40gjUR8y2Rjxa6dpG2GXHbPfvMs8ct+Lh1GH45x28Rw3Ry5 + 3mm+oAXjyQ86OnDkZ5N8lYbggD4O3w6M6pAvLkhk95AndTrifbIFPNU8PPMO7Oyr + FAHqgDsznjPFmTOtCEcN2Z1FpWgchwuYLPL+Wokqltd11nqqzi+bJ9cvSKADYdUA + AN5WUtzdpiy6LbTgSxP7ociU4Tn0g5I6aDZJ7A8Lzo0KSyZYoA485mqcO0GVAdVw + 9lq4aOT9v6d+nb4bnNkQVklLQ3fVAvJm+xdDOp9LCNCN48V2pnDOkFV6+U9nV5oy + c6XI2wIDAQABAoIBAQCUB8ip+kJiiZVKF8AqfB/aUP0jTAqOQewK1kKJ/iQCXBCq + pbo360gvdt05H5VZ/RDVkEgO2k73VSsbulqezKs8RFs2tEmU+JgTI9MeQJPWcP6X + aKy6LIYs0E2cWgp8GADgoBs8llBq0UhX0KffglIeek3n7Z6Gt4YFge2TAcW2WbN4 + XfK7lupFyo6HHyWRiYHMMARQXLJeOSdTn5aMBP0PO4bQyk5ORxTUSeOciPJUFktQ + HkvGbym7KryEfwH8Tks0L7WhzyP60PL3xS9FNOJi9m+zztwYIXGDQuKM2GDsITeD + 2mI2oHoPMyAD0wdI7BwSVW18p1h+jgfc4dlexKYRAoGBAOVfuiEiOchGghV5vn5N + RDNscAFnpHj1QgMr6/UG05RTgmcLfVsI1I4bSkbrIuVKviGGf7atlkROALOG/xRx + DLadgBEeNyHL5lz6ihQaFJLVQ0u3U4SB67J0YtVO3R6lXcIjBDHuY8SjYJ7Ci6Z6 + vuDcoaEujnlrtUhaMxvSfcUJAoGBAMPsCHXte1uWNAqYad2WdLjPDlKtQJK1diCm + rqmB2g8QE99hDOHItjDBEdpyFBKOIP+NpVtM2KLhRajjcL9Ph8jrID6XUqikQuVi + 4J9FV2m42jXMuioTT13idAILanYg8D3idvy/3isDVkON0X3UAVKrgMEne0hJpkPL + FYqgetvDAoGBAKLQ6JZMbSe0pPIJkSamQhsehgL5Rs51iX4m1z7+sYFAJfhvN3Q/ + OGIHDRp6HjMUcxHpHw7U+S1TETxePwKLnLKj6hw8jnX2/nZRgWHzgVcY+sPsReRx + NJVf+Cfh6yOtznfX00p+JWOXdSY8glSSHJwRAMog+hFGW1AYdt7w80XBAoGBAImR + NUugqapgaEA8TrFxkJmngXYaAqpA0iYRA7kv3S4QavPBUGtFJHBNULzitydkNtVZ + 3w6hgce0h9YThTo/nKc+OZDZbgfN9s7cQ75x0PQCAO4fx2P91Q+mDzDUVTeG30mE + t2m3S0dGe47JiJxifV9P3wNBNrZGSIF3mrORBVNDAoGBAI0QKn2Iv7Sgo4T/XjND + dl2kZTXqGAk8dOhpUiw/HdM3OGWbhHj2NdCzBliOmPyQtAr770GITWvbAI+IRYyF + S7Fnk6ZVVVHsxjtaHy1uJGFlaZzKR4AGNaUTOJMs6NadzCmGPAxNQQOCqoUjn4XR + rOjr9w349JooGXhOxbu8nOxX + -----END PRIVATE KEY-----""" + + lazy val pkcs8K: PKCS8KeySpec[AsymmetricKeyAlg] = + getPrivateKeySpec(priv, AsymmetricKeyAlg.RSA_PSS_Key).get + + val keyIdStr = "http://localhost:8080/rfcKey#" + // val keyUrl: ll.Url = ll.Url("http://127.0.0.1:8080/rfcKey") + val keyUrl = URI(keyIdStr) + lazy val signerF: IO[ByteVector => IO[ByteVector]] = + Signer[IO].build(pkcs8K, bobcats.AsymmetricKeyAlg.`rsa-pss-sha512`) + + import org.w3.banana.jena.io.JenaRDFReader.given + import org.w3.banana.jena.io.JenaRDFWriter.given + + lazy val keyIdData = new KeyData[IO](KeyId(Rfc8941.SfString(keyIdStr)), signerF) + + given dec: RDFDecoders[IO, R] = new RDFDecoders() + import org.http4s.syntax.all.uri + + def ioStr(uri: H4Uri): IO[String] = + emberAuthClient.flatMap(_.expect[String](uri)) + + /** Ember Client able to authenticate with above keyId */ + def emberAuthClient: IO[Client[IO]] = + EmberClientBuilder.default[IO].build.use { (client: Client[IO]) => + import org.http4s.client.middleware.Logger + val loggedClient: Client[IO] = + Logger[IO](true, true, logAction = Some(str => IO(System.out.println(str))))(client) + + val bw = new BasicWallet[IO, R]( + Map(), + Seq(keyIdData) + )(loggedClient) + + IO(AuthNClient[IO].apply(bw)(loggedClient)) + } + + def fetch(uriStr: String = "http://localhost:8080/protected/README"): String = + // ioStr(uri"http://localhost:8080/").unsafeRunSync() + // ioStr(uri"http://localhost:8080/protected/").unsafeRunSync() + ioStr(H4Uri.unsafeFromString(uriStr)).unsafeRunSync() + +end AnHttpSigClient diff --git a/scripts/jvm/src/main/scala/scripts/Fetch.sc b/scripts/jvm/src/main/scala/scripts/Fetch.sc deleted file mode 100644 index f4686ca..0000000 --- a/scripts/jvm/src/main/scala/scripts/Fetch.sc +++ /dev/null @@ -1,76 +0,0 @@ -import _root_.io.lemonlabs.uri as ll -import bobcats.util.BouncyJavaPEMUtils.getPrivateKeySpec -import bobcats.{AsymmetricKeyAlg, PKCS8KeySpec, Signer, Verifier} -import cats.effect.* -import cats.effect.unsafe.IORuntime -import net.bblfish.app.auth.AuthNClient -import net.bblfish.wallet.{BasicId, BasicWallet} -import org.w3.banana.jena.JenaRdf -import org.w3.banana.jena.JenaRdf.ops -import ops.given -import org.http4s.Uri as H4Uri -import org.w3.banana.http4sIO.RDFDecoders -import org.w3.banana.jena.JenaRdf.R -import scodec.bits.ByteVector -implicit val runtime: IORuntime = cats.effect.unsafe.IORuntime.global -import org.http4s.client.* -import org.http4s.ember.client.* - -val priv = """-----BEGIN PRIVATE KEY----- - MIIEvgIBADALBgkqhkiG9w0BAQoEggSqMIIEpgIBAAKCAQEAr4tmm3r20Wd/Pbqv - P1s2+QEtvpuRaV8Yq40gjUR8y2Rjxa6dpG2GXHbPfvMs8ct+Lh1GH45x28Rw3Ry5 - 3mm+oAXjyQ86OnDkZ5N8lYbggD4O3w6M6pAvLkhk95AndTrifbIFPNU8PPMO7Oyr - FAHqgDsznjPFmTOtCEcN2Z1FpWgchwuYLPL+Wokqltd11nqqzi+bJ9cvSKADYdUA - AN5WUtzdpiy6LbTgSxP7ociU4Tn0g5I6aDZJ7A8Lzo0KSyZYoA485mqcO0GVAdVw - 9lq4aOT9v6d+nb4bnNkQVklLQ3fVAvJm+xdDOp9LCNCN48V2pnDOkFV6+U9nV5oy - c6XI2wIDAQABAoIBAQCUB8ip+kJiiZVKF8AqfB/aUP0jTAqOQewK1kKJ/iQCXBCq - pbo360gvdt05H5VZ/RDVkEgO2k73VSsbulqezKs8RFs2tEmU+JgTI9MeQJPWcP6X - aKy6LIYs0E2cWgp8GADgoBs8llBq0UhX0KffglIeek3n7Z6Gt4YFge2TAcW2WbN4 - XfK7lupFyo6HHyWRiYHMMARQXLJeOSdTn5aMBP0PO4bQyk5ORxTUSeOciPJUFktQ - HkvGbym7KryEfwH8Tks0L7WhzyP60PL3xS9FNOJi9m+zztwYIXGDQuKM2GDsITeD - 2mI2oHoPMyAD0wdI7BwSVW18p1h+jgfc4dlexKYRAoGBAOVfuiEiOchGghV5vn5N - RDNscAFnpHj1QgMr6/UG05RTgmcLfVsI1I4bSkbrIuVKviGGf7atlkROALOG/xRx - DLadgBEeNyHL5lz6ihQaFJLVQ0u3U4SB67J0YtVO3R6lXcIjBDHuY8SjYJ7Ci6Z6 - vuDcoaEujnlrtUhaMxvSfcUJAoGBAMPsCHXte1uWNAqYad2WdLjPDlKtQJK1diCm - rqmB2g8QE99hDOHItjDBEdpyFBKOIP+NpVtM2KLhRajjcL9Ph8jrID6XUqikQuVi - 4J9FV2m42jXMuioTT13idAILanYg8D3idvy/3isDVkON0X3UAVKrgMEne0hJpkPL - FYqgetvDAoGBAKLQ6JZMbSe0pPIJkSamQhsehgL5Rs51iX4m1z7+sYFAJfhvN3Q/ - OGIHDRp6HjMUcxHpHw7U+S1TETxePwKLnLKj6hw8jnX2/nZRgWHzgVcY+sPsReRx - NJVf+Cfh6yOtznfX00p+JWOXdSY8glSSHJwRAMog+hFGW1AYdt7w80XBAoGBAImR - NUugqapgaEA8TrFxkJmngXYaAqpA0iYRA7kv3S4QavPBUGtFJHBNULzitydkNtVZ - 3w6hgce0h9YThTo/nKc+OZDZbgfN9s7cQ75x0PQCAO4fx2P91Q+mDzDUVTeG30mE - t2m3S0dGe47JiJxifV9P3wNBNrZGSIF3mrORBVNDAoGBAI0QKn2Iv7Sgo4T/XjND - dl2kZTXqGAk8dOhpUiw/HdM3OGWbhHj2NdCzBliOmPyQtAr770GITWvbAI+IRYyF - S7Fnk6ZVVVHsxjtaHy1uJGFlaZzKR4AGNaUTOJMs6NadzCmGPAxNQQOCqoUjn4XR - rOjr9w349JooGXhOxbu8nOxX - -----END PRIVATE KEY-----""" - -val pkcs8K: PKCS8KeySpec[AsymmetricKeyAlg] = - getPrivateKeySpec(priv, AsymmetricKeyAlg.RSA_PSS_Key).get -//val keyUrl: ll.Url = ll.Url("http://127.0.0.1:8080/rfcKey") -val keyUrl = URI("http://localhost:8080/rfcKey#") -val signerF: IO[ByteVector => IO[ByteVector]] = - Signer[IO].build(pkcs8K, bobcats.AsymmetricKeyAlg.`rsa-pss-sha512`) -import org.w3.banana.jena.io.JenaRDFReader.given -import org.w3.banana.jena.io.JenaRDFWriter.given - -val keyid = net.bblfish.wallet.KeyData[IO, R](keyUrl, signerF) - -given dec: RDFDecoders[IO, R] = new RDFDecoders() -import org.http4s.syntax.all.uri - -def ioStr(uri: H4Uri): IO[String] = - EmberClientBuilder.default[IO].build.use { (client: Client[IO]) => - import org.http4s.client.middleware.Logger - given loggedClient: Client[IO] = - Logger[IO](true, true, logAction = Some(str => IO(System.out.println(str))))(client) - val bw = BasicWallet[IO, R]( - Map(), - Seq(keyid) - ) - val newClient: Client[IO] = AuthNClient[IO].apply(bw)(loggedClient) - newClient.expect[String](uri) - } -//ioStr(uri"http://localhost:8080/").unsafeRunSync() -//ioStr(uri"http://localhost:8080/protected/").unsafeRunSync() -ioStr(uri"http://localhost:8080/protected/README").unsafeRunSync() diff --git a/scripts/jvm/src/main/scala/scripts/MiniCF.scala b/scripts/jvm/src/main/scala/scripts/MiniCF.scala index ae0c733..b1eadeb 100644 --- a/scripts/jvm/src/main/scala/scripts/MiniCF.scala +++ b/scripts/jvm/src/main/scala/scripts/MiniCF.scala @@ -27,14 +27,16 @@ object MiniCF: @main def crawlContainer(stream: String = "http://localhost:8080/ldes/miniCityFlows/stream#"): Unit = val ioStr: IO[fs2.Stream[IO, RDF.Graph[JR]]] = - EmberClientBuilder.default[IO].build.use { (client: Client[IO]) => + AnHttpSigClient.emberAuthClient.flatMap { (client: Client[IO]) => given web: Web[IO, JR] = new H4Web[IO, JR](client) val spider: LdesSpider[IO, JR] = new LdesSpider[IO, JR] val streamUri: RDF.URI[JR] = ops.URI(stream) spider.crawl(streamUri) } + val l: IO[List[RDF.Graph[JR]]] = - fs2.Stream.eval(ioStr).flatten + fs2.Stream.eval(ioStr) + .flatten .reduce((g1, g2) => g1.union(g2)) .compile[IO, IO, RDF.Graph[JR]] .toList diff --git a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala index 6f36ad0..d7271b8 100644 --- a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala +++ b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala @@ -28,14 +28,14 @@ import io.lemonlabs.uri.Url import run.cosy.http.auth.MessageSignature import net.bblfish.app.Wallet import net.bblfish.web.util.SecurityPrefix -import net.bblfish.web.util.UrlUtil.{http4sUrlToLLUrl, llUrltoHttp4s} +import run.cosy.web.util.UrlUtil.{http4sUrlToLLUrl, llUrltoHttp4s} import org.http4s as h4s import org.http4s.client.Client import org.http4s.{Challenge, headers as h4hdr} import org.http4s.headers.Authorization import org.w3.banana.Ops import org.w3.banana.RDF -import org.w3.banana.http4sIO.RDFDecoders +import run.cosy.ld.http4s.RDFDecoders import org.w3.banana.io.JsonLd import org.w3.banana.io.RDFReader import org.w3.banana.io.Turtle From 9c18d67dcc33ca363e82c2069353e26db8575e61 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Thu, 16 Mar 2023 08:10:01 +0100 Subject: [PATCH 09/42] don't send Accept headers 2c acl req + reformat --- .../net/bblfish/wallet/BasicAuthWallet.scala | 105 ++++++++---------- 1 file changed, 45 insertions(+), 60 deletions(-) diff --git a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala index d7271b8..b54818a 100644 --- a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala +++ b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala @@ -18,45 +18,36 @@ package net.bblfish.wallet import bobcats.* import cats.data.NonEmptyList -import cats.effect.Clock -import cats.effect.Concurrent -import cats.effect.IO +import cats.effect.{Clock, Concurrent, IO} import cats.implicits.* import com.apicatalog.jsonld.uri.UriUtils import io.lemonlabs.uri as ll import io.lemonlabs.uri.Url -import run.cosy.http.auth.MessageSignature import net.bblfish.app.Wallet import net.bblfish.web.util.SecurityPrefix -import run.cosy.web.util.UrlUtil.{http4sUrlToLLUrl, llUrltoHttp4s} import org.http4s as h4s import org.http4s.client.Client -import org.http4s.{Challenge, headers as h4hdr} import org.http4s.headers.Authorization -import org.w3.banana.Ops -import org.w3.banana.RDF -import run.cosy.ld.http4s.RDFDecoders -import org.w3.banana.io.JsonLd -import org.w3.banana.io.RDFReader -import org.w3.banana.io.Turtle -import org.w3.banana.prefix.Cert -import org.w3.banana.prefix.WebACL +import org.http4s.{headers as h4hdr, Challenge} +import org.w3.banana.io.{JsonLd, RDFReader, Turtle} import org.w3.banana.prefix.WebACL.apply +import org.w3.banana.prefix.{Cert, WebACL} +import org.w3.banana.{Ops, RDF} import run.cosy.http.Http -import run.cosy.http.headers.{ReqSigInput, Rfc8941, SigInput} -import run.cosy.http.headers.Rfc8941.IList -import run.cosy.http.headers.Rfc8941.{SfInt, SfString} +import run.cosy.http.auth.MessageSignature +import run.cosy.http.headers.Rfc8941.{IList, SfInt, SfString} import run.cosy.http.headers.SigIn.KeyId +import run.cosy.http.headers.{ReqSigInput, Rfc8941, SigInput} import run.cosy.http.messages.{NoServerContext, ReqSelectors} import run.cosy.http4s.Http4sTp.HT as H4 import run.cosy.http4s.messages.SelectorFnsH4 - -import scala.reflect.TypeTest -import scala.util.Failure -import scala.util.Try +import run.cosy.ld.http4s.RDFDecoders +import run.cosy.web.util.UrlUtil.{http4sUrlToLLUrl, llUrltoHttp4s} import scodec.bits.ByteVector import scala.concurrent.duration.FiniteDuration +import scala.reflect.TypeTest +import scala.util.{Failure, Try} class BasicId(val username: String, val password: String) @@ -79,54 +70,47 @@ class KeyData[F[_]]( end KeyData - trait ChallengeResponse: - //the challenge scheme for which this response is designed - def forChallengeScheme: String - - // - def respondTo[F[_]]( - remote: ll.AbsoluteUrl, // request for original Host - originalRequest: h4s.Request[F], - response: h4s.Response[F] - ): F[h4s.Request[F]] + // the challenge scheme for which this response is designed + def forChallengeScheme: String -/** - * First attempt at a Wallet, just to get things going. - * The wallet must be given collections of passwords for domains, KeyIDs for - * http signatures, cookies (!), OpenId info, ... - * - * Note that CookieJar is a algebra for a mutable used to produce a middleware. - * So perhaps a wallet takes a set of middlewares - * each adapted for a particular situation, and it would proceed as follows: + // + def respondTo[F[_]]( + remote: ll.AbsoluteUrl, // request for original Host + originalRequest: h4s.Request[F], + response: h4s.Response[F] + ): F[h4s.Request[F]] + +/** First attempt at a Wallet, just to get things going. The wallet must be given collections of + * passwords for domains, KeyIDs for http signatures, cookies (!), OpenId info, ... * + * Note that CookieJar is a algebra for a mutable used to produce a middleware. So perhaps a wallet + * takes a set of middlewares each adapted for a particular situation, and it would proceed as + * follows: * - * it needs a client to follow links to the WAC rules, though it may be better - * if instead it were given a DataSet proxied to the web to allow caching. - * On the other hand with free monads one could have those be interpreted according - * to context... Todo: compare this way of working with free-monads. + * it needs a client to follow links to the WAC rules, though it may be better if instead it were + * given a DataSet proxied to the web to allow caching. On the other hand with free monads one + * could have those be interpreted according to context... Todo: compare this way of working with + * free-monads. */ class BasicWallet[F[_], Rdf <: RDF]( - db: Map[ll.Authority, BasicId], - keyIdDB: Seq[KeyData[F]] -)(client: Client[F])( - using - ops: Ops[Rdf], - rdfDecoders: RDFDecoders[F, Rdf], - fc: Concurrent[F], - clock: Clock[F] + db: Map[ll.Authority, BasicId], + keyIdDB: Seq[KeyData[F]] +)(client: Client[F])(using + ops: Ops[Rdf], + rdfDecoders: RDFDecoders[F, Rdf], + fc: Concurrent[F], + clock: Clock[F] ) extends Wallet[F]: val reqSel: ReqSelectors[H4] = new ReqSelectors[H4](using new SelectorFnsH4()) + import ops.{*, given} + import rdfDecoders.allrdf import reqSel.* import reqSel.RequestHd.* - import run.cosy.http4s.Http4sTp import run.cosy.http4s.Http4sTp.{*, given} - import ops.* - import ops.given - import rdfDecoders.allrdf import scala.language.implicitConversions def reqToH4Req(h4req: h4s.Request[F]): Http.Request[H4] = @@ -183,8 +167,7 @@ class BasicWallet[F[_], Rdf <: RDF]( client .fetchAs[RDF.rGraph[Rdf]]( h4s.Request( - uri = absLink.withoutFragment, - headers = h4s.Headers(rdfDecoders.allRdfAccept) + uri = absLink.withoutFragment ) ) .flatMap { (rG: RDF.rGraph[Rdf]) => @@ -267,9 +250,11 @@ class BasicWallet[F[_], Rdf <: RDF]( case Some(h4hdr.`WWW-Authenticate`(nel)) => // do we recognise a method? for url <- fc.fromTry(Try(http4sUrlToLLUrl(lastReq.uri).toAbsoluteUrl)) - authdReq <- fc.fromTry(basicChallenge(url.authority, lastReq, nel)).handleErrorWith { _ => - httpSigChallenge(url, lastReq, failed, nel) - } + authdReq <- fc + .fromTry(basicChallenge(url.authority, lastReq, nel)) + .handleErrorWith { _ => + httpSigChallenge(url, lastReq, failed, nel) + } yield authdReq case _ => ??? // fail end sign From e0f718cbedf0ecc43985950cc68108560ea3632d Mon Sep 17 00:00:00 2001 From: Henry Story Date: Thu, 16 Mar 2023 16:56:48 +0100 Subject: [PATCH 10/42] seperate the crawling and the extracting --- .../main/scala/run/cosy/ldes/LdesSpider.scala | 26 +++--------- .../jvm/src/main/scala/scripts/MiniCF.scala | 40 +++++++++++++++++-- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/ldes/shared/src/main/scala/run/cosy/ldes/LdesSpider.scala b/ldes/shared/src/main/scala/run/cosy/ldes/LdesSpider.scala index a2a587a..9fd20f4 100644 --- a/ldes/shared/src/main/scala/run/cosy/ldes/LdesSpider.scala +++ b/ldes/shared/src/main/scala/run/cosy/ldes/LdesSpider.scala @@ -25,7 +25,7 @@ class LdesSpider[F[_]: Concurrent, R <: RDF]( /** given the ldes stream URL, crawl the nodes of that stream */ - def crawl(stream: RDF.URI[R]): F[fs2.Stream[F, RDF.Graph[R]]] = + def crawl(stream: RDF.URI[R]): F[fs2.Stream[F, fs2.Chunk[UriNGraph[R]]]] = import cats.syntax.all.toFunctorOps import cats.syntax.flatMap.toFlatMapOps for @@ -35,15 +35,13 @@ class LdesSpider[F[_]: Concurrent, R <: RDF]( /* This crawls pages forward and collects all tree.member observations... * This is very much tied to a particular use of ldes - * todo: it should be able to fetch the shEx to find out what to - * collect, or an pattern object should be passed. * @returns an fs2.Stream of such observation mini graphs **/ def crawlNodesForward( stream: RDF.URI[R], startNodes: Seq[PNGraph[R]], visitedRef: Ref[F, Set[RDF.URI[R]]] - ): fs2.Stream[F, RDF.Graph[R]] = + ): fs2.Stream[F, fs2.Chunk[UriNGraph[R]]] = import cats.syntax.all.toFlatMapOps fs2.Stream.unfoldLoopEval(startNodes) { nodes => import cats.syntax.traverse.{*, given} @@ -68,26 +66,12 @@ class LdesSpider[F[_]: Concurrent, R <: RDF]( .rel(tree.node) .collect { case ung: UriNGraph[R] => ung } // we need to place the pointer on the Collection of each page - val collInPages: Seq[UriNGraph[R]] = - pages.map(ung => new UriNGraph[R](stream, ung.name, ung.graph)) - val obs: RDF.Graph[R] = collInPages - .rel(tree.member) - .map(png => - png.collect( - rdf.typ, - wgs84.location, - sosa.hasSimpleResult, - sosa.madeBySensor, - sosa.observedProperty, - sosa.resultTime - )() - ) - .fold(Graph.empty)((g1, g2) => g1 union g2) ( - obs, + fs2.Chunk(pages*), if nextPages.isEmpty then None else Some(nextPages) ) } end crawlNodesForward - + + diff --git a/scripts/jvm/src/main/scala/scripts/MiniCF.scala b/scripts/jvm/src/main/scala/scripts/MiniCF.scala index b1eadeb..87b7ce5 100644 --- a/scripts/jvm/src/main/scala/scripts/MiniCF.scala +++ b/scripts/jvm/src/main/scala/scripts/MiniCF.scala @@ -2,7 +2,8 @@ package scripts // This fetches data from a mini City Flows web server import cats.effect.unsafe.IORuntime -import cats.effect.{unsafe, Concurrent, IO, Ref} +import cats.effect.{Concurrent, IO, Ref, unsafe} +import fs2.Chunk import org.http4s.EntityDecoder import org.http4s.client.Client import org.http4s.ember.client.* @@ -26,22 +27,55 @@ object MiniCF: @main def crawlContainer(stream: String = "http://localhost:8080/ldes/miniCityFlows/stream#"): Unit = - val ioStr: IO[fs2.Stream[IO, RDF.Graph[JR]]] = + val streamUri: RDF.URI[JR] = ops.URI(stream) + + val ioStr: IO[fs2.Stream[IO, Chunk[UriNGraph[JR]]]] = AnHttpSigClient.emberAuthClient.flatMap { (client: Client[IO]) => given web: Web[IO, JR] = new H4Web[IO, JR](client) val spider: LdesSpider[IO, JR] = new LdesSpider[IO, JR] - val streamUri: RDF.URI[JR] = ops.URI(stream) spider.crawl(streamUri) } val l: IO[List[RDF.Graph[JR]]] = fs2.Stream.eval(ioStr) .flatten + .unchunks + .map( uriNG => selectObs(uriNG, streamUri) ) + .unchunks .reduce((g1, g2) => g1.union(g2)) .compile[IO, IO, RDF.Graph[JR]] .toList l.unsafeRunSync().foreach(_.triples.foreach(println)) end crawlContainer + + import run.cosy.ldes.prefix as ldesPre + val foaf = prefix.FOAF[JR] + val tree = ldesPre.TREE[JR] + val sosa = ldesPre.SOSA[JR] + val wgs84 = ldesPre.WGS84[JR] + val ldes = ldesPre.LDES[JR] + + /** select the observations in tree:Node of the ldes:Stream + * + * todo: it should be able to be given or fetch the shEx described in the + * ldes:Stream to decide what to select + * */ + def selectObs(page: UriNGraph[JR], streamUrl: RDF.URI[JR]): fs2.Chunk[RDF.Graph[JR]] = + val collInPage = new UriNGraph[JR](streamUrl, page.name, page.graph) + val obs: Seq[RDF.Graph[JR]] = collInPage + .rel(tree.member) + .map(png => + png.collect( + rdf.typ, + wgs84.location, + sosa.hasSimpleResult, + sosa.madeBySensor, + sosa.observedProperty, + sosa.resultTime + )() + ) + fs2.Chunk(obs *) + // .fold(Graph.empty)((g1, g2) => g1 union g2) end MiniCF From 5c401cc547891cdb072e2a6b98ec701b527e0de2 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Sat, 6 May 2023 16:31:41 +0200 Subject: [PATCH 11/42] Add initial support for default ACLs --- .../net/bblfish/app/auth/AuthNClient.scala | 7 +- .../cosy/solid/app/auth/AuthNClientTest.scala | 51 +++--- build.sbt | 6 +- .../scala/run/cosy/web/util/UrlUtil.scala | 13 ++ .../src/main/scala/run/cosy/ld/PNGraph.scala | 26 ++- .../main/scala/run/cosy/ld/http4s/H4Web.scala | 29 +++- .../main/scala/run/cosy/ldes/LdesSpider.scala | 29 +++- .../scala/run/cosy/ldes/prefix/LDES.scala | 19 ++- .../scala/run/cosy/ldes/prefix/SOSA.scala | 18 ++- .../scala/run/cosy/ldes/prefix/TREE.scala | 20 ++- .../scala/run/cosy/ldes/prefix/WGS84.scala | 18 ++- .../test/scala/run/cosy/ld/FoafWebTest.scala | 99 +++++++----- .../test/scala/run/cosy/ld/JenaWebTest.scala | 19 ++- .../scala/run/cosy/ld/LdesBrokenWebTest.scala | 28 +++- .../scala/run/cosy/ld/LdesSimpleWebTest.scala | 20 ++- .../test/scala/run/cosy/ld/MiniFoafWWW.scala | 58 ++++--- .../run/cosy/ld/ldes/BrokenMiniLdesWWW.scala | 24 ++- .../scala/run/cosy/ld/ldes/MiniLdesWWW.scala | 34 ++-- project/Dependencies.scala | 4 +- .../main/scala/scripts/AnHttpSigClient.scala | 21 ++- .../jvm/src/main/scala/scripts/MiniCF.scala | 40 +++-- .../main/scala/net/bblfish/app/Wallet.scala | 6 + .../main/scala/net/bblfish/wallet/AuthN.scala | 36 +++-- .../net/bblfish/wallet/BasicAuthWallet.scala | 148 ++++++++++++------ .../net/bblfish/web/util/SecurityPrefix.scala | 2 +- .../scala/net/bblfish/wallet/HttpTests.scala | 56 +++++++ .../test/scala/net/bblfish/wallet/Jena.scala | 24 +++ .../net/bblfish/wallet/WallletToolsTest.scala | 94 +++++++++++ 28 files changed, 739 insertions(+), 210 deletions(-) create mode 100644 wallet/shared/src/test/scala/net/bblfish/wallet/HttpTests.scala create mode 100644 wallet/shared/src/test/scala/net/bblfish/wallet/Jena.scala create mode 100644 wallet/shared/src/test/scala/net/bblfish/wallet/WallletToolsTest.scala diff --git a/authn/shared/src/main/scala/net/bblfish/app/auth/AuthNClient.scala b/authn/shared/src/main/scala/net/bblfish/app/auth/AuthNClient.scala index 9ca435c..ed68d81 100644 --- a/authn/shared/src/main/scala/net/bblfish/app/auth/AuthNClient.scala +++ b/authn/shared/src/main/scala/net/bblfish/app/auth/AuthNClient.scala @@ -47,6 +47,7 @@ object AuthNClient: hotswap: Hotswap[F, Response[F]] ): F[Response[F]] = hotswap.clear *> // Release the prior connection before allocating a new + // todo: we should enhance the req with a signature if we already have info on the server hotswap.swap(client.run(req)).flatMap { (resp: Response[F]) => // todo: may want a lot more flexibility than attempt numbering to determine if we should retry or not. resp.status match @@ -59,7 +60,11 @@ object AuthNClient: // using the pattern from FollowRedirect example using Hotswap. // Not 100% sure this is so much needed here... Hotswap.create[F, Response[F]].flatMap { hotswap => - Resource.eval(authLoop(req, 0, hotswap)) + Resource.eval( + wallet.signFromDB(req).flatMap { possiblySignedReq => + authLoop(possiblySignedReq, 0, hotswap) + } + ) } } end apply diff --git a/authn/shared/src/test/scala/run/cosy/solid/app/auth/AuthNClientTest.scala b/authn/shared/src/test/scala/run/cosy/solid/app/auth/AuthNClientTest.scala index 0205ea1..346b5ab 100644 --- a/authn/shared/src/test/scala/run/cosy/solid/app/auth/AuthNClientTest.scala +++ b/authn/shared/src/test/scala/run/cosy/solid/app/auth/AuthNClientTest.scala @@ -22,8 +22,7 @@ import cats.syntax.all.* import io.lemonlabs import io.lemonlabs.uri.config.UriConfig import net.bblfish.app.auth.AuthNClient -import net.bblfish.wallet.BasicWallet -import net.bblfish.wallet.BasicId +import net.bblfish.wallet.{BasicId, BasicWallet, WalletTools} import org.http4s.Uri.Host import org.http4s.client.Client import org.http4s.client.dsl.Http4sClientDsl @@ -33,10 +32,20 @@ import org.http4s.headers.* import org.http4s.server.middleware.authentication.BasicAuth import org.http4s.server.{AuthMiddleware, Router} import org.http4s.syntax.all.* -import org.http4s.{AuthedRoutes, BasicCredentials, Challenge, Headers, HttpRoutes, Request, Response, Status, Uri} +import org.http4s.{ + AuthedRoutes, + BasicCredentials, + Challenge, + Headers, + HttpRoutes, + Request, + Response, + Status, + Uri +} import org.typelevel.ci.* import org.w3.banana.jena.JenaRdf.R as Jena -import java.util.concurrent.atomic.* +import run.cosy.ld.http4s.RDFDecoders case class User(id: Long, name: String) @@ -142,27 +151,27 @@ class AuthNClientTest extends munit.CatsEffectSuite { // test with client now val defaultClient: Client[IO] = Client.fromHttpApp(routes.orNotFound) given UriConfig = UriConfig.default + import org.w3.banana.jena.JenaRdf.ops + import org.w3.banana.io.{JsonLd, RDFXML, RelRDFReader, Turtle} + import org.w3.banana.jena.io.JenaRDFReader.given + given rdfDecoders: RDFDecoders[IO, Jena] = new run.cosy.ld.http4s.RDFDecoders[IO, Jena] // val logedClient: Client[IO] = ResponseLogger[IO](true, true, logAction = Some(s => IO(println(s))))(defaultClient) - val client: Client[IO] = AuthNClient[IO]( - new BasicWallet[IO,Jena]( - Map( - lemonlabs.uri.Authority("localhost") -> - BasicId(username, password) - ) + val wallet1 = new BasicWallet[IO, Jena]( + Map( + lemonlabs.uri.Authority("localhost") -> + BasicId(username, password) ) )(defaultClient) - - val clientBad: Client[IO] = AuthNClient[IO]( - BasicWallet[IO,Jena]( - Map( - lemonlabs.uri.Authority("localhost") -> BasicId( - username, - password + "bad" - ) - ), - List() + val client: Client[IO] = AuthNClient[IO](wallet1)(defaultClient) + val wallet2 = BasicWallet[IO, Jena]( + Map( + lemonlabs.uri.Authority("localhost") -> BasicId( + username, + password + "bad" + ) ) - )(defaultClient) + )(client) + val clientBad: Client[IO] = AuthNClient[IO](wallet2)(defaultClient) test("Wallet Based Auth") { client.get(uri"http://localhost/auth") { (res: Response[IO]) => diff --git a/build.sbt b/build.sbt index 78017d7..f73833e 100644 --- a/build.sbt +++ b/build.sbt @@ -114,7 +114,7 @@ lazy val ldes = crossProject(JVMPlatform) resolvers += sonatypeSNAPSHOT, libraryDependencies ++= Seq( // cats.effect.value, - cats.fs2.value, + cats.fs2.value ), libraryDependencies ++= Seq( munit.value % Test, @@ -135,11 +135,11 @@ lazy val ioExt4s = crossProject(JVMPlatform) resolvers += sonatypeSNAPSHOT, libraryDependencies ++= Seq( http4s.client.value, - banana.bananaIO.value, + banana.bananaIO.value ), libraryDependencies ++= Seq( munit.value % Test, - cats.munitEffect.value % Test, + cats.munitEffect.value % Test ) ) diff --git a/ioExt4s/shared/src/main/scala/run/cosy/web/util/UrlUtil.scala b/ioExt4s/shared/src/main/scala/run/cosy/web/util/UrlUtil.scala index 25a81b6..caeb438 100644 --- a/ioExt4s/shared/src/main/scala/run/cosy/web/util/UrlUtil.scala +++ b/ioExt4s/shared/src/main/scala/run/cosy/web/util/UrlUtil.scala @@ -19,8 +19,21 @@ package run.cosy.web.util import com.comcast.ip4s import io.lemonlabs.uri as ll import org.http4s.Uri as h4Uri +import org.w3.banana.{Ops, RDF} + +import scala.util.Try object UrlUtil { + + extension (h4uri: org.http4s.Uri) def toLL: ll.Url = http4sUrlToLLUrl(h4uri) + + extension (llUri: ll.Url) def toh4: org.http4s.Uri = llUrltoHttp4s(llUri) + + extension [R <: RDF](uri: RDF.URI[R])(using ops: Ops[R]) + def toLL: Try[ll.Uri] = + import ops.{*, given} + ll.Uri.parseTry(uri.value) + // ignoring username:password urls def http4sUrlToLLUrl(u: org.http4s.Uri): ll.Url = import u.{host as h4host, path as h4path, query as h4query, port as h4port, scheme as h4scheme} diff --git a/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala b/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala index 9ab19f1..7080a6e 100644 --- a/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala +++ b/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package run.cosy.ld import cats.MonadError @@ -9,10 +25,10 @@ import org.w3.banana.{Ops, RDF} trait Web[F[_]: Concurrent, R <: RDF]: /** get a Graph for the given URL (without a fragment) */ def get(url: RDF.URI[R]): F[RDF.Graph[R]] - -/** get Pointed Named Graph for given url */ + + /** get Pointed Named Graph for given url */ def getPNG(url: RDF.URI[R]): F[UriNGraph[R]] - + /** A Pointed Named Graph, ie, a pointer into a NamedGraph We don't use a case class here as * equality between PNGgraphs is complicated by the need to prove isomorphism between graphs, and * the nodes have to be equivalent. @@ -31,9 +47,9 @@ trait PNGraph[R <: RDF]: * graph */ def collect( - forward: RDF.URI[R]* + forward: RDF.URI[R]* )( - backward: RDF.URI[R]* + backward: RDF.URI[R]* )(using ops: Ops[R]): RDF.Graph[R] = import ops.{*, given} val f: Seq[Seq[RDF.Triple[R]]] = diff --git a/ldes/shared/src/main/scala/run/cosy/ld/http4s/H4Web.scala b/ldes/shared/src/main/scala/run/cosy/ld/http4s/H4Web.scala index 4f48d7f..3f20505 100644 --- a/ldes/shared/src/main/scala/run/cosy/ld/http4s/H4Web.scala +++ b/ldes/shared/src/main/scala/run/cosy/ld/http4s/H4Web.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package run.cosy.ld.http4s import cats.effect.Concurrent @@ -10,13 +26,10 @@ import run.cosy.ld.http4s.RDFDecoders import run.cosy.ld.{UriNGraph, Web} import run.cosy.web.util.UrlUtil.http4sUrlToLLUrl -/** - * Web implementation in http4s - * todo: we need a client that can find out about redirects, so that we can - * correctly name the resource - * todo: we also need a web cache that can be queried and updated (or perhaps that would use - * this) - */ +/** Web implementation in http4s todo: we need a client that can find out about redirects, so that + * we can correctly name the resource todo: we also need a web cache that can be queried and + * updated (or perhaps that would use this) + */ class H4Web[F[_]: Concurrent, R <: RDF]( client: Client[F] )(using @@ -36,7 +49,7 @@ class H4Web[F[_]: Concurrent, R <: RDF]( ) yield rG.resolveAgainst(http4sUrlToLLUrl(doc).toAbsoluteUrl) - //todo: this should really use client.fetchPNG + // todo: this should really use client.fetchPNG override def getPNG(url: RDF.URI[R]): F[UriNGraph[R]] = val doc = url.fragmentLess get(doc).map(g => UriNGraph(url, doc, g)) diff --git a/ldes/shared/src/main/scala/run/cosy/ldes/LdesSpider.scala b/ldes/shared/src/main/scala/run/cosy/ldes/LdesSpider.scala index 9fd20f4..82c8fdd 100644 --- a/ldes/shared/src/main/scala/run/cosy/ldes/LdesSpider.scala +++ b/ldes/shared/src/main/scala/run/cosy/ldes/LdesSpider.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package run.cosy.ldes import cats.effect.{IO, Ref} @@ -8,9 +24,9 @@ import run.cosy.ld.{PNGraph, UriNGraph, Web} import scala.collection.immutable.Set -class LdesSpider[F[_]: Concurrent, R <: RDF]( - using www: Web[F, R], - ops: Ops[R] +class LdesSpider[F[_]: Concurrent, R <: RDF](using + www: Web[F, R], + ops: Ops[R] ): import ops.{*, given} @@ -22,9 +38,8 @@ class LdesSpider[F[_]: Concurrent, R <: RDF]( val sosa = ldesPre.SOSA[R] val wgs84 = ldesPre.WGS84[R] val ldes = ldesPre.LDES[R] - - - /** given the ldes stream URL, crawl the nodes of that stream */ + + /** given the ldes stream URL, crawl the nodes of that stream */ def crawl(stream: RDF.URI[R]): F[fs2.Stream[F, fs2.Chunk[UriNGraph[R]]]] = import cats.syntax.all.toFunctorOps import cats.syntax.flatMap.toFlatMapOps @@ -73,5 +88,3 @@ class LdesSpider[F[_]: Concurrent, R <: RDF]( ) } end crawlNodesForward - - diff --git a/ldes/shared/src/main/scala/run/cosy/ldes/prefix/LDES.scala b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/LDES.scala index 54bac8f..ae36007 100644 --- a/ldes/shared/src/main/scala/run/cosy/ldes/prefix/LDES.scala +++ b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/LDES.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package run.cosy.ldes.prefix import org.w3.banana.{Ops, PrefixBuilder, RDF} @@ -6,7 +22,7 @@ object LDES: def apply[Rdf <: RDF](using ops: Ops[Rdf]) = new LDES() class LDES[Rdf <: RDF](using ops: Ops[Rdf]) - extends PrefixBuilder[Rdf]("ldes", ops.URI("https://w3id.org/ldes#")): + extends PrefixBuilder[Rdf]("ldes", ops.URI("https://w3id.org/ldes#")): val EventStream = apply("EventStream") val EventSource = apply("EventSource") @@ -20,4 +36,3 @@ class LDES[Rdf <: RDF](using ops: Ops[Rdf]) val timestampPath = apply("timestampPath") val versionMaterializationOf = apply("versionMaterializationOf") val versionMaterializationUntil = apply("versionMaterializationUntil") - diff --git a/ldes/shared/src/main/scala/run/cosy/ldes/prefix/SOSA.scala b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/SOSA.scala index b38f62a..2a7b9d3 100644 --- a/ldes/shared/src/main/scala/run/cosy/ldes/prefix/SOSA.scala +++ b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/SOSA.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package run.cosy.ldes.prefix import org.w3.banana.{Ops, PrefixBuilder, RDF} @@ -6,7 +22,7 @@ object SOSA: def apply[Rdf <: RDF](using ops: Ops[Rdf]) = new SOSA() class SOSA[Rdf <: RDF](using ops: Ops[Rdf]) - extends PrefixBuilder[Rdf]("tree", ops.URI("http://www.w3.org/ns/sosa/")): + extends PrefixBuilder[Rdf]("tree", ops.URI("http://www.w3.org/ns/sosa/")): val FeatureOfInterest = apply("FeatureOfInterest") val ObservableProperty = apply("ObservableProperty") diff --git a/ldes/shared/src/main/scala/run/cosy/ldes/prefix/TREE.scala b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/TREE.scala index 786c272..6b4f6d0 100644 --- a/ldes/shared/src/main/scala/run/cosy/ldes/prefix/TREE.scala +++ b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/TREE.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package run.cosy.ldes.prefix import org.w3.banana.{Ops, PrefixBuilder, RDF} @@ -6,7 +22,7 @@ object TREE: def apply[Rdf <: RDF](using ops: Ops[Rdf]) = new TREE() class TREE[Rdf <: RDF](using ops: Ops[Rdf]) - extends PrefixBuilder[Rdf]("tree", ops.URI("https://w3id.org/tree#")): + extends PrefixBuilder[Rdf]("tree", ops.URI("https://w3id.org/tree#")): val Collection = apply("Collection") val ViewDescription = apply("ViewDescription") @@ -38,5 +54,3 @@ class TREE[Rdf <: RDF](using ops: Ops[Rdf]) val longitudeTile = apply("longitudeTile") val latitudeTile = apply("latitudeTile") val timeQuery = apply("timeQuery") - - \ No newline at end of file diff --git a/ldes/shared/src/main/scala/run/cosy/ldes/prefix/WGS84.scala b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/WGS84.scala index d7553a2..a63ac10 100644 --- a/ldes/shared/src/main/scala/run/cosy/ldes/prefix/WGS84.scala +++ b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/WGS84.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package run.cosy.ldes.prefix import org.w3.banana.{Ops, PrefixBuilder, RDF} @@ -6,7 +22,7 @@ object WGS84: def apply[R <: RDF](using Ops[R]) = new WGS84() class WGS84[R <: RDF](using ops: Ops[R]) - extends PrefixBuilder[R]("wgs84", ops.URI("http://www.w3.org/2003/01/geo/wgs84_pos#")): + extends PrefixBuilder[R]("wgs84", ops.URI("http://www.w3.org/2003/01/geo/wgs84_pos#")): lazy val Point = apply("Point") lazy val SpatialThing = apply("SpatialThing") diff --git a/ldes/shared/src/test/scala/run/cosy/ld/FoafWebTest.scala b/ldes/shared/src/test/scala/run/cosy/ld/FoafWebTest.scala index 7a6acce..163c7a2 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/FoafWebTest.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/FoafWebTest.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package run.cosy.ld import cats.effect.IO @@ -10,77 +26,88 @@ import run.cosy.ld.MiniFoafWWW.EricP trait FoafWebTest[R <: RDF]()(using ops: Ops[R]) extends CatsEffectSuite { val miniWeb = new MiniFoafWWW[R] import miniWeb.foaf - given www: Web[IO,R] = miniWeb + given www: Web[IO, R] = miniWeb import ops.{*, given} - import PNGraph.{given,*} + import PNGraph.{*, given} import MiniFoafWWW.* import cats.effect.IO.asyncForIO - + test("find bbl friends WebIDs inside graph (no jump)") { - + val bblng: IO[UriNGraph[R]] = www.getPNG(URI(Bbl)) bblng.map { bblNg => - //todo: make it possible to just get nodes - val kns: Seq[String] = (bblNg/foaf.knows).collect{ - case u: UriNGraph[R] => u.point.value + // todo: make it possible to just get nodes + val kns: Seq[String] = (bblNg / foaf.knows).collect { case u: UriNGraph[R] => + u.point.value } - assertEquals(Set(kns*), Set(EricP,Timbl,CSarven)) + assertEquals(Set(kns*), Set(EricP, Timbl, CSarven)) } - + } - + test("find bblFriend WebIDs after jumping") { www.getPNG(URI(Bbl)).flatMap { bblNg => val friendsDef: fs2.Stream[IO, PNGraph[R]] = (bblNg / foaf.knows).jump val expectedURIs: Set[String] = Set(EricP, Timbl, CSarven) val result1: IO[Set[String]] = friendsDef.compile.toList.map(pnglst => val uris: Seq[String] = pnglst.collect { case u: UriNGraph[R] => u.point.value } - Set(uris *) + Set(uris*) ) result1.map(set => assertEquals(set, expectedURIs)) } - - //todo: test what happens if a WebID is broken. + + // todo: test what happens if a WebID is broken. } - + test("find canonical names of friends (as defined in their profile)") { www.getPNG(URI(Bbl)).flatMap { bblNg => - //most of the names are only available from the definitional graphs - val names: fs2.Stream[IO, String] = (bblNg / foaf.knows).jump.collect { - case ug: UriNGraph[R] => fs2.Stream(ug / foaf.name *) - }.flatten.collect { case litG: LiteralNGraph[R] => litG.point.text } + // most of the names are only available from the definitional graphs + val names: fs2.Stream[IO, String] = (bblNg / foaf.knows).jump + .collect { case ug: UriNGraph[R] => + fs2.Stream(ug / foaf.name*) + } + .flatten + .collect { case litG: LiteralNGraph[R] => litG.point.text } names.compile.toList.map(lst => - assertEquals(Set(lst*), Set( - "Tim Berners-Lee", "Eric Prud'hommeaux", "Sarven Capadisli"))) + assertEquals(Set(lst*), Set("Tim Berners-Lee", "Eric Prud'hommeaux", "Sarven Capadisli")) + ) } } - - + test("find all names of friends (local and remote)") { www.getPNG(URI(Bbl)).flatMap { bblNg => - //most of the names are only available from the definitional graphs - val allNames: fs2.Stream[IO, String] = (bblNg / foaf.knows).jump.collect { - case ug: SubjPNGraph[R] => fs2.Stream(ug / foaf.name *) - }.flatten.collect { case litG: LiteralNGraph[R] => litG.point.text } - + // most of the names are only available from the definitional graphs + val allNames: fs2.Stream[IO, String] = (bblNg / foaf.knows).jump + .collect { case ug: SubjPNGraph[R] => + fs2.Stream(ug / foaf.name*) + } + .flatten + .collect { case litG: LiteralNGraph[R] => litG.point.text } + allNames.compile.toList.map { lst => - assertEquals(Set(lst*), Set( - "Tim Berners-Lee", "Eric Prud'hommeaux", "Sarven Capadisli", "James Gosling")) + assertEquals( + Set(lst*), + Set("Tim Berners-Lee", "Eric Prud'hommeaux", "Sarven Capadisli", "James Gosling") + ) } } } - + test("find all names of friends (local and remote) - shorter version") { www.getPNG(URI(Bbl)).flatMap { bblNg => - //most of the names are only available from the definitional graphs + // most of the names are only available from the definitional graphs val allNames: fs2.Stream[IO, String] = - bblNg.rel(foaf.knows).jump - .rel(foaf.name) - .collect { case litG: LiteralNGraph[R] => litG.point.text } + bblNg + .rel(foaf.knows) + .jump + .rel(foaf.name) + .collect { case litG: LiteralNGraph[R] => litG.point.text } allNames.compile.toList.map { lst => - assertEquals(Set(lst *), Set( - "Tim Berners-Lee", "Eric Prud'hommeaux", "Sarven Capadisli", "James Gosling")) + assertEquals( + Set(lst*), + Set("Tim Berners-Lee", "Eric Prud'hommeaux", "Sarven Capadisli", "James Gosling") + ) } } } diff --git a/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala b/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala index 8fd8b3a..87755ed 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala @@ -1,12 +1,25 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package run.cosy.ld import org.w3.banana.jena.JenaRdf.{R, given} - class JenaFoafWebTest extends FoafWebTest[R]() class JenaLdesSimpleWebTest extends LdesSimpleWebTest[R]() class JenaLdesBrokenWebTest extends LdesBrokenWebTest[R]() - - diff --git a/ldes/shared/src/test/scala/run/cosy/ld/LdesBrokenWebTest.scala b/ldes/shared/src/test/scala/run/cosy/ld/LdesBrokenWebTest.scala index d2421d8..78bd9d5 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/LdesBrokenWebTest.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/LdesBrokenWebTest.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package run.cosy.ld import cats.effect.IO @@ -28,13 +44,11 @@ trait LdesBrokenWebTest[R <: RDF]()(using ops: Ops[R]) extends CatsEffectSuite: for v <- visitedRef.get // here we make sure we don't visit the same page twice and we don't fail on missing pages - pagesEither <- views - .collect { - case ung: UriNGraph[R] if !v.contains(ung.point.fragmentLess) => - ung.jump[IO].attempt - } - .sequence - pages = pagesEither.collect{ case Right(png) => png } + pagesEither <- views.collect { + case ung: UriNGraph[R] if !v.contains(ung.point.fragmentLess) => + ung.jump[IO].attempt + }.sequence + pages = pagesEither.collect { case Right(png) => png } _ <- visitedRef.update { v => val urls = pages.map(_.name) v.union(urls.toSet) diff --git a/ldes/shared/src/test/scala/run/cosy/ld/LdesSimpleWebTest.scala b/ldes/shared/src/test/scala/run/cosy/ld/LdesSimpleWebTest.scala index db4cac9..2491c95 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/LdesSimpleWebTest.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/LdesSimpleWebTest.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package run.cosy.ld import cats.effect.IO @@ -92,8 +108,8 @@ trait LdesSimpleWebTest[R <: RDF]()(using ops: Ops[R]) extends CatsEffectSuite: import cats.syntax.traverse.{*, given} val x: Seq[IO[UriNGraph[R]]] = views.collect { case ung: UriNGraph[R] => ung.jump } - //note: a problem with x.sequence is that it would need all UriNGraphs to be complete - //before the IO is complete. What if some get stuck? + // note: a problem with x.sequence is that it would need all UriNGraphs to be complete + // before the IO is complete. What if some get stuck? val pagesIO: IO[Seq[UriNGraph[R]]] = x.sequence // val pagesIO: IO[Seq[UriNGraph[R]]] = // views.collect({ case ung: UriNGraph[R] => ung.jump }).sequence diff --git a/ldes/shared/src/test/scala/run/cosy/ld/MiniFoafWWW.scala b/ldes/shared/src/test/scala/run/cosy/ld/MiniFoafWWW.scala index 5fe4472..0a0a4ed 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/MiniFoafWWW.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/MiniFoafWWW.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package run.cosy.ld import cats.Id @@ -9,14 +25,14 @@ import org.w3.banana.diesel import org.w3.banana.diesel.{*, given} object MiniFoafWWW: - val BblCard = "https://bblfish.net/people/henry/card" - val Bbl = BblCard + "#me" - val TimblCard = "https://www.w3.org/People/Berners-Lee/card" - val Timbl = TimblCard + "#i" - val EricPCard = "https://www.w3.org/People/Eric/ericP-foaf.rdf" - val EricP = EricPCard + "#ericP" - val CSarvenCard = "https://csarven.ca/" - val CSarven = CSarvenCard + "#i" + val BblCard = "https://bblfish.net/people/henry/card" + val Bbl = BblCard + "#me" + val TimblCard = "https://www.w3.org/People/Berners-Lee/card" + val Timbl = TimblCard + "#i" + val EricPCard = "https://www.w3.org/People/Eric/ericP-foaf.rdf" + val EricP = EricPCard + "#ericP" + val CSarvenCard = "https://csarven.ca/" + val CSarven = CSarvenCard + "#i" class MiniFoafWWW[R <: RDF](using ops: Ops[R]) extends Web[IO, R]: import ops.{*, given} @@ -25,7 +41,7 @@ class MiniFoafWWW[R <: RDF](using ops: Ops[R]) extends Web[IO, R]: val foaf = prefix.FOAF[R] val fr = Lang("fr") // todo, put together a list of Lang constants val en = Lang("en") - + def getPNG(url: RDF.URI[R]): IO[UriNGraph[R]] = val doc = url.fragmentLess get(doc).map(g => new UriNGraph(url, doc, g)) @@ -39,31 +55,25 @@ class MiniFoafWWW[R <: RDF](using ops: Ops[R]) extends Web[IO, R]: -- foaf.knows ->- URI(EricP) -- foaf.knows ->- URI(CSarven) -- foaf.knows ->- URI(Timbl) - -- foaf.knows ->- ( BNode() -- foaf.name ->- "James Gosling") - ).graph + -- foaf.knows ->- (BNode() -- foaf.name ->- "James Gosling")).graph case TimblCard => (rURI("#i") -- foaf.name ->- "Tim Berners-Lee".lang(en) - -- foaf.knows ->- URI(Bbl) - -- foaf.knows ->- URI(CSarven) - -- foaf.knows ->- URI(EricP) - -- foaf.knows ->- ( BNode() -- foaf.name ->- "Vint Cerf") - ).graph + -- foaf.knows ->- URI(Bbl) + -- foaf.knows ->- URI(CSarven) + -- foaf.knows ->- URI(EricP) + -- foaf.knows ->- (BNode() -- foaf.name ->- "Vint Cerf")).graph case EricPCard => (rURI("#ericP") -- foaf.name ->- "Eric Prud'hommeaux".lang(fr) -- foaf.knows ->- URI(Bbl) -- foaf.knows ->- URI(CSarven) - -- foaf.knows ->- URI(Timbl) - ).graph + -- foaf.knows ->- URI(Timbl)).graph case CSarvenCard => (rURI("#i") -- foaf.name ->- "Sarven Capadisli".lang(en) -- foaf.knows ->- URI(Bbl) -- foaf.knows ->- URI(Timbl) - -- foaf.knows ->- URI(EricP) - ).graph - - + -- foaf.knows ->- URI(EricP)).graph + case _ => ops.rGraph.empty - + IO(res.resolveAgainst(AbsoluteUrl.parse(url.value))) end get - \ No newline at end of file diff --git a/ldes/shared/src/test/scala/run/cosy/ld/ldes/BrokenMiniLdesWWW.scala b/ldes/shared/src/test/scala/run/cosy/ld/ldes/BrokenMiniLdesWWW.scala index c4e7e6f..dbc3524 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/ldes/BrokenMiniLdesWWW.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/ldes/BrokenMiniLdesWWW.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package run.cosy.ld.ldes import cats.effect.IO @@ -43,8 +59,8 @@ class BrokenMiniLdesWWW[R <: RDF](using ops: Ops[R]) extends MiniLdesWWW[R]: ).graph.triples.toSeq Some(g) case Collection => - // we add a link to a non existing view - super.getRelativeGraph(url).map{rg => - rg + rTriple(rURI(""), tree.view, rURI("2021-09-20")) - } + // we add a link to a non existing view + super.getRelativeGraph(url).map { rg => + rg + rTriple(rURI(""), tree.view, rURI("2021-09-20")) + } case _ => super.getRelativeGraph(url) diff --git a/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala b/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala index d90c1b5..531c522 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package run.cosy.ld.ldes import cats.effect.IO @@ -5,7 +21,7 @@ import io.lemonlabs.uri.AbsoluteUrl import org.w3.banana.diesel.{*, given} import org.w3.banana.{diesel, *} import run.cosy.ld.* -import run.cosy.ld.ldes.prefix as ldesPre +import run.cosy.ldes.prefix as ldesPre import run.cosy.ldes.prefix.{LDES, SOSA, TREE, WGS84} import scala.language.implicitConversions @@ -154,14 +170,14 @@ class MiniLdesWWW[R <: RDF](using ops: Ops[R]) extends Web[IO, R]: -- tree.path ->- sosa.resultTime -- tree.value ->- ("2021-09-06T00:00:00+02" ^^ xsd.dateTimeStamp) )).graph ++ obsrvs(D09_05).flatten ++ ( - rURI(".").a(ldes.EventStream) - -- ldes.timestampPath ->- sosa.resultTime - -- tree.shape ->- rURI("flows-shacl") - -- tree.view ->- rURI("") - -- tree.member ->- rURI("#3") - -- tree.member ->- rURI("#482") - -- tree.member ->- rURI("#4464") - ).graph.triples.toSeq + rURI(".").a(ldes.EventStream) + -- ldes.timestampPath ->- sosa.resultTime + -- tree.shape ->- rURI("flows-shacl") + -- tree.view ->- rURI("") + -- tree.member ->- rURI("#3") + -- tree.member ->- rURI("#482") + -- tree.member ->- rURI("#4464") + ).graph.triples.toSeq case D09_06 => (rURI("").a(tree.Node) -- tree.relation ->- ( diff --git a/project/Dependencies.scala b/project/Dependencies.scala index da06ccd..e642856 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -5,14 +5,14 @@ object Dependencies { object Ver { val scala = "3.2.2" val http4s = "1.0.0-M39" - val banana = "0.9-c996591-SNAPSHOT" + val banana = "0.9-c996591-SNAPSHOT" val bobcats = "0.3-3236e64-SNAPSHOT" val httpSig = "0.4-ac23f8b-SNAPSHOT" } object other { // https://github.com/lemonlabsuk/scala-uri - val scalaUri = Def.setting("io.lemonlabs" %%% "scala-uri" % "4.0.2") + val scalaUri = Def.setting("io.lemonlabs" %%% "scala-uri" % "4.0.3") } // https://http4s.org/v1.0/client/ object http4s { diff --git a/scripts/jvm/src/main/scala/scripts/AnHttpSigClient.scala b/scripts/jvm/src/main/scala/scripts/AnHttpSigClient.scala index 98e81a0..8dfb9dd 100644 --- a/scripts/jvm/src/main/scala/scripts/AnHttpSigClient.scala +++ b/scripts/jvm/src/main/scala/scripts/AnHttpSigClient.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package scripts import _root_.io.lemonlabs.uri as ll @@ -6,7 +22,7 @@ import bobcats.{AsymmetricKeyAlg, PKCS8KeySpec, Signer, Verifier} import cats.effect.* import cats.effect.unsafe.IORuntime import net.bblfish.app.auth.AuthNClient -import net.bblfish.wallet.{BasicId, BasicWallet, KeyData} +import net.bblfish.wallet.{BasicId, BasicWallet, KeyData, WalletTools} import org.w3.banana.jena.JenaRdf import org.w3.banana.jena.JenaRdf.ops import ops.given @@ -53,7 +69,7 @@ object AnHttpSigClient: lazy val pkcs8K: PKCS8KeySpec[AsymmetricKeyAlg] = getPrivateKeySpec(priv, AsymmetricKeyAlg.RSA_PSS_Key).get - + val keyIdStr = "http://localhost:8080/rfcKey#" // val keyUrl: ll.Url = ll.Url("http://127.0.0.1:8080/rfcKey") val keyUrl = URI(keyIdStr) @@ -67,6 +83,7 @@ object AnHttpSigClient: given dec: RDFDecoders[IO, R] = new RDFDecoders() import org.http4s.syntax.all.uri + given wt: WalletTools[R] = new WalletTools[R] def ioStr(uri: H4Uri): IO[String] = emberAuthClient.flatMap(_.expect[String](uri)) diff --git a/scripts/jvm/src/main/scala/scripts/MiniCF.scala b/scripts/jvm/src/main/scala/scripts/MiniCF.scala index 87b7ce5..2003350 100644 --- a/scripts/jvm/src/main/scala/scripts/MiniCF.scala +++ b/scripts/jvm/src/main/scala/scripts/MiniCF.scala @@ -1,8 +1,24 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package scripts // This fetches data from a mini City Flows web server import cats.effect.unsafe.IORuntime -import cats.effect.{Concurrent, IO, Ref, unsafe} +import cats.effect.{unsafe, Concurrent, IO, Ref} import fs2.Chunk import org.http4s.EntityDecoder import org.http4s.client.Client @@ -13,6 +29,7 @@ import run.cosy.ld.http4s.{H4Web, RDFDecoders} import run.cosy.ld.{PNGraph, UriNGraph, Web} import run.cosy.ldes.LdesSpider +// Mini City Flows object MiniCF: type JR = org.w3.banana.jena.JenaRdf.type @@ -28,24 +45,26 @@ object MiniCF: @main def crawlContainer(stream: String = "http://localhost:8080/ldes/miniCityFlows/stream#"): Unit = val streamUri: RDF.URI[JR] = ops.URI(stream) - + val ioStr: IO[fs2.Stream[IO, Chunk[UriNGraph[JR]]]] = AnHttpSigClient.emberAuthClient.flatMap { (client: Client[IO]) => given web: Web[IO, JR] = new H4Web[IO, JR](client) val spider: LdesSpider[IO, JR] = new LdesSpider[IO, JR] spider.crawl(streamUri) } - + val l: IO[List[RDF.Graph[JR]]] = - fs2.Stream.eval(ioStr) + fs2.Stream + .eval(ioStr) .flatten .unchunks - .map( uriNG => selectObs(uriNG, streamUri) ) + .map(uriNG => selectObs(uriNG, streamUri)) .unchunks .reduce((g1, g2) => g1.union(g2)) .compile[IO, IO, RDF.Graph[JR]] .toList l.unsafeRunSync().foreach(_.triples.foreach(println)) + end crawlContainer import run.cosy.ldes.prefix as ldesPre @@ -56,10 +75,10 @@ object MiniCF: val ldes = ldesPre.LDES[JR] /** select the observations in tree:Node of the ldes:Stream - * - * todo: it should be able to be given or fetch the shEx described in the - * ldes:Stream to decide what to select - * */ + * + * todo: it should be able to be given or fetch the shEx described in the ldes:Stream to decide + * what to select + */ def selectObs(page: UriNGraph[JR], streamUrl: RDF.URI[JR]): fs2.Chunk[RDF.Graph[JR]] = val collInPage = new UriNGraph[JR](streamUrl, page.name, page.graph) val obs: Seq[RDF.Graph[JR]] = collInPage @@ -74,8 +93,7 @@ object MiniCF: sosa.resultTime )() ) - fs2.Chunk(obs *) + fs2.Chunk(obs*) // .fold(Graph.empty)((g1, g2) => g1 union g2) - end MiniCF diff --git a/wallet/shared/src/main/scala/net/bblfish/app/Wallet.scala b/wallet/shared/src/main/scala/net/bblfish/app/Wallet.scala index fd418aa..100b0f5 100644 --- a/wallet/shared/src/main/scala/net/bblfish/app/Wallet.scala +++ b/wallet/shared/src/main/scala/net/bblfish/app/Wallet.scala @@ -29,4 +29,10 @@ trait Wallet[F[_]] { * [[https://github.com/http4s/http4s/discussions/5930#discussioncomment-3777066 cats-uri discussion]] */ def sign(failed: Response[F], lastReq: Request[F]): F[Request[F]] + + /** previous requests to a server will return acls and methods that can be assumed to be valid + * @param req + * @return + */ + def signFromDB(req: Request[F]): F[Request[F]] } diff --git a/wallet/shared/src/main/scala/net/bblfish/wallet/AuthN.scala b/wallet/shared/src/main/scala/net/bblfish/wallet/AuthN.scala index f57f0f7..6c5a8b1 100644 --- a/wallet/shared/src/main/scala/net/bblfish/wallet/AuthN.scala +++ b/wallet/shared/src/main/scala/net/bblfish/wallet/AuthN.scala @@ -1,27 +1,41 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package net.bblfish.wallet import cats.Id import org.http4s.Request import org.http4s.headers.Authorization -/** - * The signature for authenticating a Request. +/** The signature for authenticating a Request. * - * @tparam F The monad for the Stream - * @tparam S ASync monad for time it takes to get signature - could be Id monad on many platforms + * @tparam F + * The monad for the Stream + * @tparam S + * ASync monad for time it takes to get signature - could be Id monad on many platforms */ trait AuthN[F[_], S[_]]: - /** Signing the request - * This could be a function, but on some platforms and for some signing algorithms - * the signing may need to be asynchronous in S[_]. The internal monad F on the other + /** Signing the request This could be a function, but on some platforms and for some signing + * algorithms the signing may need to be asynchronous in S[_]. The internal monad F on the other * had will be asyncrhonous, as it is needed for streaming responses. */ def sign(originalReq: Request[F]): S[Request[F]] - - + class Basic[F[_]](username: String, pass: String) extends AuthN[F, Id]: - override - def sign(originalReq: Request[F]): Id[Request[F]] = + override def sign(originalReq: Request[F]): Id[Request[F]] = originalReq.withHeaders( Authorization(org.http4s.BasicCredentials(username, pass)) ) diff --git a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala index b54818a..e780ecd 100644 --- a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala +++ b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala @@ -20,19 +20,20 @@ import bobcats.* import cats.data.NonEmptyList import cats.effect.{Clock, Concurrent, IO} import cats.implicits.* -import com.apicatalog.jsonld.uri.UriUtils import io.lemonlabs.uri as ll import io.lemonlabs.uri.Url import net.bblfish.app.Wallet import net.bblfish.web.util.SecurityPrefix import org.http4s as h4s import org.http4s.client.Client -import org.http4s.headers.Authorization -import org.http4s.{headers as h4hdr, Challenge} +import org.http4s.headers.{Authorization, Link, LinkValue} +import org.http4s.{headers as h4hdr, Challenge, Request, Uri} +import org.typelevel.ci.CIString +import org.w3.banana.RDF.Statement as St import org.w3.banana.io.{JsonLd, RDFReader, Turtle} import org.w3.banana.prefix.WebACL.apply import org.w3.banana.prefix.{Cert, WebACL} -import org.w3.banana.{Ops, RDF} +import org.w3.banana.{prefix, Ops, RDF} import run.cosy.http.Http import run.cosy.http.auth.MessageSignature import run.cosy.http.headers.Rfc8941.{IList, SfInt, SfString} @@ -81,6 +82,75 @@ trait ChallengeResponse: response: h4s.Response[F] ): F[h4s.Request[F]] +object BasicWallet: + val effectiveAclLink = "effectiveAccessControl" + val EffectiveAclOpt = Some(effectiveAclLink) + val aclRelTypes = List(EffectiveAclOpt, "acl", effectiveAclLink) + +/** place code that only needs RDF and ops here. */ +class WalletTools[Rdf <: RDF](using ops: Ops[Rdf]): + import run.cosy.web.util.UrlUtil.* + import ops.{*, given} + + val wac: WebACL[Rdf] = WebACL[Rdf] + val foaf = prefix.FOAF[Rdf] + val sec: SecurityPrefix[Rdf] = SecurityPrefix[Rdf] + + def withinTry(requestUri: RDF.URI[Rdf], container: RDF.URI[Rdf]): Try[Boolean] = + for + case requ: ll.AbsoluteUrl <- requestUri.toLL + case ctnrU: ll.AbsoluteUrl <- container.toLL + yield + val se = requ.schemeOption == ctnrU.schemeOption + val ae = requ.authorityOption == ctnrU.authorityOption + val rp = requ.path.parts + // because ll.Url interprets the https://foo.bar/ as having a path with one empty string + val cp = ctnrU.path.parts + val cpClean = if cp.last == "" then cp.dropRight(1) else cp + val pe = rp.startsWith(cpClean) + se && ae && pe + + extension (uri: RDF.URI[Rdf]) + def contains(longer: RDF.URI[Rdf]): Boolean = + withinTry(longer, uri).getOrElse(false) + + def findAclsFor(g: RDF.Graph[Rdf], requestUri: RDF.URI[Rdf]): Iterator[St.Subject[Rdf]] = + import run.cosy.web.util.UrlUtil.* + val directRules: Iterator[St.Subject[Rdf]] = g.find(`*`, wac.accessTo, requestUri).map(_.subj) + val defaultRules: Iterator[St.Subject[Rdf]] = g.find(`*`, wac.default, `*`).collect { + case ops.Triple(rule, _, defaultContainer: RDF.URI[Rdf]) + if defaultContainer.contains(requestUri) => + rule + } + directRules ++ defaultRules + + def modeForMethod(method: h4s.Method): RDF.URI[Rdf] = + import h4s.Method.{GET, HEAD, SEARCH} + if List(GET, HEAD, SEARCH).contains(method) then wac.Read + else wac.Write + + def findAgents( + aclGr: RDF.Graph[Rdf], + reqUrl: RDF.URI[Rdf], + mode: h4s.Method + ): Iterator[St.Object[Rdf]] = { + for + ruleNode <- findAclsFor(aclGr, reqUrl) + if !aclGr + .find(ruleNode, wac.mode, modeForMethod(mode)) + .isEmpty + obj <- aclGr + .find(ruleNode, wac.agent, `*`) + .map(_.obj) + .collect[St.Subject[Rdf]] { + case u: RDF.URI[Rdf] => u + case b: RDF.BNode[Rdf] => b + // todo: the key could be in a literal too !? + } + yield obj + } +end WalletTools + /** First attempt at a Wallet, just to get things going. The wallet must be given collections of * passwords for domains, KeyIDs for http signatures, cookies (!), OpenId info, ... * @@ -95,13 +165,14 @@ trait ChallengeResponse: */ class BasicWallet[F[_], Rdf <: RDF]( db: Map[ll.Authority, BasicId], - keyIdDB: Seq[KeyData[F]] + keyIdDB: Seq[KeyData[F]] = Seq() )(client: Client[F])(using ops: Ops[Rdf], rdfDecoders: RDFDecoders[F, Rdf], fc: Concurrent[F], clock: Clock[F] ) extends Wallet[F]: + val wt: WalletTools[Rdf] = new WalletTools[Rdf] val reqSel: ReqSelectors[H4] = new ReqSelectors[H4](using new SelectorFnsH4()) import ops.{*, given} @@ -112,6 +183,7 @@ class BasicWallet[F[_], Rdf <: RDF]( import run.cosy.http4s.Http4sTp.{*, given} import scala.language.implicitConversions + import wt.* def reqToH4Req(h4req: h4s.Request[F]): Http.Request[H4] = h4req.asInstanceOf[Http.Request[H4]] @@ -119,9 +191,6 @@ class BasicWallet[F[_], Rdf <: RDF]( def h4ReqToHttpReq(h4req: Http.Request[H4]): h4s.Request[F] = h4req.asInstanceOf[h4s.Request[F]] - val WAC: WebACL[Rdf] = WebACL[Rdf] - val sec: SecurityPrefix[Rdf] = SecurityPrefix[Rdf] - // todo: here we assume the request Uri usually has the Host. Need to verify, or pass the full Url def basicChallenge( host: ll.Authority, // request for original Host @@ -146,24 +215,23 @@ class BasicWallet[F[_], Rdf <: RDF]( response: h4s.Response[F], nel: NonEmptyList[h4s.Challenge] ): F[h4s.Request[F]] = - val aclLinks: Seq[h4hdr.LinkValue] = + import BasicWallet.* + val aclLinks: List[LinkValue] = for httpSig <- nel.find(_.scheme == "HttpSig").toList - link <- response.headers.get[h4hdr.Link].toList + link <- response.headers.get[Link].toList linkVal <- link.values.toList - if linkVal.rel == Some("acl") // todo: also check other wac url + rel <- linkVal.rel.toList + if aclRelTypes.contains(rel) yield linkVal - - aclLinks.headOption match // todo: we try to the first only but what if... + aclLinks + .find(_.rel == EffectiveAclOpt) + .orElse(aclLinks.headOption) match case None => - fc.raiseError( - Exception( - Exception("no acl Link in header. Cannot find where the rules are.") - ) - ) - case Some(link) => + fc.raiseError(Exception("no acl Link in header. Cannot find where the rules are.")) + case Some(linkVal) => val h4req = llUrltoHttp4s(requestUrl) - val absLink: h4s.Uri = h4req.resolve(link.uri) + val absLink: h4s.Uri = h4req.resolve(linkVal.uri) client .fetchAs[RDF.rGraph[Rdf]]( h4s.Request( @@ -177,32 +245,25 @@ class BasicWallet[F[_], Rdf <: RDF]( val reqRes = ops.URI(originalRequest.uri.toString) // <-- // this requires all the info to be in the same graph. Needs generalisation to // jump across graphs - val mode = modeForMethod(originalRequest.method) - val keyNodes = for - tr <- g.find(`*`, WAC.accessTo, reqRes) - if !g - .find(tr.subj, WAC.mode, mode) - .isEmpty // todo: too simple - obj <- g - .find(tr.subj, WAC.agent, `*`) - .map(_.obj) - .collect[RDF.Statement.Object[Rdf]] { - case u: RDF.URI[Rdf] => u - case b: RDF.BNode[Rdf] => b - // todo: the key could be in a literal too !? - } - tr3 <- g.find(*, sec.controller, obj) - yield tr3.subj + val keyNodes: Iterator[St.Subject[Rdf]] = for + agentNode <- findAgents(g, reqRes, originalRequest.method) + controllerTriple <- g.find(*, sec.controller, agentNode) + yield controllerTriple.subj + import run.cosy.http4s.Http4sTp.given - val keys: List[KeyData[F]] = keyNodes.toList.collect { case u: RDF.URI[Rdf] => - keyIdDB.find(kid => kid.keyIdAtt.value.asciiStr == u.value).toList - }.flatten + val keys: Iterable[KeyData[F]] = keyNodes + .collect { case u: RDF.URI[Rdf] => + keyIdDB.find(kid => kid.keyIdAtt.value.asciiStr == u.value).toList + } + .flatten + .to(Iterable) for keydt <- fc.fromOption[KeyData[F]]( keys.headOption, Exception( - s"none of our keys fit the ACL $lllink for resource $reqRes accessed in $mode matches the rules in { $g } " + s"none of our keys fit the ACL $lllink for resource $reqRes accessed in " + + s"${originalRequest.method} matches the rules in { $g } " ) ) signingFn <- keydt.signer @@ -220,11 +281,6 @@ class BasicWallet[F[_], Rdf <: RDF]( } end httpSigChallenge - def modeForMethod(method: h4s.Method): RDF.URI[Rdf] = - import h4s.Method.{GET, HEAD, SEARCH} - if List(GET, HEAD, SEARCH).contains(method) then WAC.Read - else WAC.Write - /** This is different from middleware such as FollowRedirects, as that essentially continues the * request. Here we need to stop the request and make new ones to find the access control rules * for the given resource. (that could just be a BasicAuth request for a password, or a more @@ -259,4 +315,6 @@ class BasicWallet[F[_], Rdf <: RDF]( case _ => ??? // fail end sign + override def signFromDB(req: h4s.Request[F]): F[h4s.Request[F]] = fc.point(req) + end BasicWallet diff --git a/wallet/shared/src/main/scala/net/bblfish/web/util/SecurityPrefix.scala b/wallet/shared/src/main/scala/net/bblfish/web/util/SecurityPrefix.scala index 076383f..4438067 100644 --- a/wallet/shared/src/main/scala/net/bblfish/web/util/SecurityPrefix.scala +++ b/wallet/shared/src/main/scala/net/bblfish/web/util/SecurityPrefix.scala @@ -24,7 +24,7 @@ object SecurityPrefix: /** Note: the security prefix https://w3id.org/security/v1# is not a namespace! That is a context * document for rdfa, containing shortcuts for many different names coming from different * namespaces. TODO: we need something like the Prefix class to model these! TODO: we have a copy - * of this in Reactive Solid. Where should this go? (actually perhaps somehwere in this repo is not + * of this in Reactive Solid. Where should this go? (actually perhaps somewhere in this repo is not * a bad idea) * @param ops * @tparam Rdf diff --git a/wallet/shared/src/test/scala/net/bblfish/wallet/HttpTests.scala b/wallet/shared/src/test/scala/net/bblfish/wallet/HttpTests.scala new file mode 100644 index 0000000..cc96bc8 --- /dev/null +++ b/wallet/shared/src/test/scala/net/bblfish/wallet/HttpTests.scala @@ -0,0 +1,56 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.bblfish.wallet + +import org.http4s.{ParseResult, Uri} +import org.http4s.headers.{Link, LinkValue} + +class HttpTests extends munit.FunSuite { + + test( + ("parse Link header. " + + "ignore because of https://github.com/http4s/http4s/issues/7101").ignore + ) { + val ht1 = """<>; rel=https://www.w3.org/ns/auth/acl#accessControl""" + val ht1res: ParseResult[Link] = Link.parse(ht1) + assert(ht1res.isLeft, ht1res) + } + + test("parse Link header working") { + val h1 = """; rel=acl,""" + + """ ; rel="https://www.w3.org/ns/auth/acl#accessControl"""" + val pr1: ParseResult[Link] = Link.parse(h1) + assert(pr1.isRight, pr1) + + val h2 = """; rel=acl,""" + + """ ; rel=https://www.w3.org/ns/auth/acl#accessControl""" + val pr2: ParseResult[Link] = Link.parse(h2) + assert(pr2.isLeft, pr2) + + val rfc8288Ex = """; rel="start http://example.net/relation/other"""" + val Right(Link(values)) = Link.parse(rfc8288Ex): @unchecked + assertEquals(values.size, 1) + assertEquals( + values.head, + LinkValue( + Uri.unsafeFromString("http://example.org/"), + rel = Some("start http://example.net/relation/other") + ) + ) + } + +} diff --git a/wallet/shared/src/test/scala/net/bblfish/wallet/Jena.scala b/wallet/shared/src/test/scala/net/bblfish/wallet/Jena.scala new file mode 100644 index 0000000..005379c --- /dev/null +++ b/wallet/shared/src/test/scala/net/bblfish/wallet/Jena.scala @@ -0,0 +1,24 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.bblfish.wallet + +import org.w3.banana.jena.JenaRdf +import org.w3.banana.jena.JenaRdf.ops +type JR = JenaRdf.type +given WalletTools[JR] = new WalletTools[JR] + +class JenaWalletToolsTest extends WalletToolsTest[JR] diff --git a/wallet/shared/src/test/scala/net/bblfish/wallet/WallletToolsTest.scala b/wallet/shared/src/test/scala/net/bblfish/wallet/WallletToolsTest.scala new file mode 100644 index 0000000..27192c6 --- /dev/null +++ b/wallet/shared/src/test/scala/net/bblfish/wallet/WallletToolsTest.scala @@ -0,0 +1,94 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.bblfish.wallet + +import org.w3.banana.{Ops, RDF} + +import scala.util.Success +import scala.language.implicitConversions +import io.lemonlabs.uri as ll +import org.http4s.Method.GET +import org.w3.banana.RDF.Statement as St + +trait WalletToolsTest[R <: RDF](using ops: Ops[R]) extends munit.FunSuite { + val wt = new WalletTools[R] + import ops.{*, given} + import wt.* + import org.w3.banana.diesel.{*, given} + + val acl1: RDF.rGraph[R] = (rURI("#R0") -- rdf.typ ->- wac.Authorization + -- wac.mode ->- wac.Control + -- wac.agent ->- BNode("a")).graph ++ ( + rURI("#R1") -- rdf.`type` ->- wac.Authorization + -- wac.mode ->- wac.Read + -- wac.default ->- rURI(".") + -- wac.agent ->- rURI("#a") // we use a URI here for testAgents + ).graph.triples.toSeq ++ ( + rURI("#R2") -- rdf.`type` ->- wac.Authorization + -- wac.mode ->- wac.Write + -- wac.default ->- rURI(".") + -- wac.agent ->- rURI("#a") + ).graph.triples.toSeq ++ ( + rURI("/rfcKey#") -- sec.controller ->- wac.Write + ).graph.triples.toSeq + + val bbl = URI("https://bblfish.net/") + val bblPpl = URI("https://bblfish.net/people") + val bblPplS = URI("https://bblfish.net/people/") + val bblCard = URI("https://bblfish.net/people/henry/card") + val alice = URI("https://alice.name/card#me") + + test("WalletTools.within") { + assertEquals(wt.withinTry(bblCard, bbl), Success(true)) + assertEquals(wt.withinTry(bbl, bblCard), Success(false)) + assertEquals(wt.withinTry(bblCard, bblPpl), Success(true)) + assertEquals(wt.withinTry(bblCard, bblPplS), Success(true)) + + assertEquals(bbl.contains(bblPpl), true) + assertEquals(bbl.contains(bblPplS), true) + assertEquals(bblPplS.contains(bblPplS), true) + assertEquals(bblPplS.contains(bblCard), true) + } + + val bblRootAcl = ll.AbsoluteUrl.parse("https://bblfish.net/.acl") + val acl1Gr: RDF.Graph[R] = acl1.resolveAgainst(bblRootAcl) + + test("test find acl") { + val r1r2: Set[St.Subject[R]] = + Set(URI("https://bblfish.net/.acl#R1"), URI("https://bblfish.net/.acl#R2")) + + val n1: Set[St.Subject[R]] = findAclsFor(acl1Gr, bbl).toSet + assertEquals(n1, r1r2) + + val n2: Set[St.Subject[R]] = findAclsFor(acl1Gr, bblPplS).toSet + assertEquals(n2, r1r2) + + val n3: Set[St.Subject[R]] = findAclsFor(acl1Gr, bblCard).toSet + assertEquals(n3, r1r2) + + val n4: List[St.Subject[R]] = findAclsFor(acl1Gr, alice).toList + assertEquals(n4, List()) + + // todo: add code for the wac: + } + + test("findAgents") { + assertEquals(findAgents(acl1Gr, bblCard, GET).toList, List(URI("https://bblfish.net/.acl#a"))) + + } + +} From fd283dc08f43d91057219431db478bb8dd1ed0a9 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Tue, 9 May 2023 18:24:54 +0200 Subject: [PATCH 12/42] README thinking through data structures 4 web cache --- .../src/main/scala/net/bblfish/web/README.md | 184 ++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 wallet/shared/src/main/scala/net/bblfish/web/README.md diff --git a/wallet/shared/src/main/scala/net/bblfish/web/README.md b/wallet/shared/src/main/scala/net/bblfish/web/README.md new file mode 100644 index 0000000..af7cb0d --- /dev/null +++ b/wallet/shared/src/main/scala/net/bblfish/web/README.md @@ -0,0 +1,184 @@ +## The Cache + +We need a cache for web resources. + +### Use ReactiveSolid's DirTree? + +We could define a Cache for a given X and Y something like + +```scala +type Cache = Map[(Protocol, Domain, port), DirTree[X, Y]] +``` + +Arguably the domain and subdomains could also be part of the dirtree, +if one wanted to think of domains hierarchically. + +Some differences: + +1. dirtree.insert removes subtrees when adding a ref for a container. + But: + * would placing the cached content in the attribute help? + * could one just rename this as `insertAndRemoveSubDirs` and create a + new method `replace` that did notchange subdirs? +2. Here we mostly do not need intermediate resources, where in akka those are + mandatory as they need to keep track of properties of the collections. + (e.g. a collection may have config info that tells the server to fetch data + from a DB) But: + * We could have `type X = Option[G]` instead of just `G` + * We do need intermediary resources if we want to keep track of + default ACLs, since we need to be able to search for resources at a domain + by path, and those are in tree form + +Improvements needed: + +1. Metadata: the container points to its ACL, + but as explained [below](#finding-the-default-acl) we may not have done a `GET` onthe container of the effective acl, + so when we work through a path hierarchy we would not have the info there to know that the container had a default + acl. A good answer would be to allow one to add incoming links to the data structure. + +## Implementations + +There are a number of ways of modelling the file structure of a server which is essentially a Tree with data at the +nodes. + +### Old OO style + +In old OO style we could just use mutable objects. + +```scala +case class DirTree[R](var ref: R, kids: scala.collection.mutable.HashMap[String, DirTree[R]]) +``` + +changing something in the tree would just require + +1. finding the node +2. setting the `ref` or `kids` var to the new value + +That requires the whole tree to be synchronised for writes if paralle access is +possible as it is in Java. + +### Using DirTree from ReactiveSolid + +In ReactiveSolid we implemented an +immutable [DirTree](https://github.com/co-operating-systems/Reactive-SoLiD/blob/30d6fd46fe30bc8be350c4a09d8592b20b791aa7/src/main/scala/run/cosy/ldp/DirTree.scala#L135). +Simplifiying just a little it is defined as: + +```scala +case class DirTree[R](ref: R, kids: scala.collection.immutable.HashMap[String, DirTree[R]]) +``` + +The library then implements its own zipper class to allow insertions and +changes to that immutable recursive data structure. + +Because it is immutable: + +* read access is easy +* write access will change the whole tree including the root. + +If a client wants access to the latest version, we need to wrap the root in a +java `AtomicReference[DirTree[R, A]]` or a cats effect `Ref` + +### Using `cats.free.Comonad[F[_],X]` + +The definition of `cats.free.Cofree` is + +```scala +final case class Cofree[S[_], A](head: A, tail: Eval[S[Cofree[S, A]]]) +``` + +If we fix `S[_]` to `Map[String,_]` as the branching Functor +and if we ignore `Eval` or just fix it to `Now`, we get an equivalent +definition as above. + +```scala +type Dir[X] = Now[Map[String, X]] +type DirTree[N] = cats.free.Cofree[Dir[_], N] +``` + +Advantages: + +* gives us all the builtin tools to work with the Directory, +* requires us to build a zipper to change the content. We could: + + re-use the one from the Reactive-Solid lib of course + + use Monocle + as [in the Solid Server demo](https://github.com/bblfish/lens-play/blob/master/src/main/scala/server/SolidServer.scala#L181) +* allows the use of `tree.coflatMap(f: DirTree[N] => A)`. Would be useful for: + * perhaps finding the ACLs linked to a directory + * ??? + +Questions: + +1. could the lazy evaluation be useful with `Eval.Later` or `Eval.Always`? + * when loading data from the file system? + Not really because if all we want is `/foo/bar/baz` we will need to load the Maps for `/foo` and `/foo/bar`, and + since we are dealing with immutable data structures we have to load all the elements of the map at once. This + actually shows the advantages of our tree of Actors, as those can only load one relation of the map when requested + by checking the file system. + +### Using `cats.effect.kernel.Ref` + +In a strongly concurrent system with many threads and potentially a lot of writes, we may want to model +the Tree as an immutable structure +using [cats.effect.kernel.Ref](https://typelevel.org/cats-effect/api/3.x/cats/effect/kernel/Ref.html). +That starts to be a lot closer to the original OO method of programming, as we no longer need a zipper to change +a node, but can just change the node itself, by for example creating a new `kids` in the parent, and replacing the +old one by updating the Ref. + +```scala +case class DirTree[X](node: X, kids: Ref[Map[String, Tree[X]]]) +``` + +On the client side, for the cache we don't really need to optimise for synchronous +access to the DB as this fetching data from a web server is restricted by the speed of +light and by the rules of politeness. + +### Using actors as a Tree + +That is actually what Akka does. Each actor has children in the form of a tree. The only reason we needed our +`DirTree` immutable data structure there was to more efficiently find references for those actors so that we did not +have to have +* all messages pass through one root actor and +* so we could reduce the message passing from paths of length n to 1 + +But the major advantage of actors is that they make it easy to check for files only when they are needed. So we don't +have to for example update the whole map by reading all the files present in one directory. We can just check that +the directory or file we need is there. (see question 1 in [using Cofree](#using-catsfreecomonadfx) above) + +## Finding the default ACL + +The client needs a cache it can use to guess for as many resources as possible what the default acl for them could be. +without going through a `401` response. After all, if a client has fetched a resource in a container and later +wants to fetch other resources in that container or subcontainer then if the rule it used to authenticate was a default +rule then it makes sense that it could continue using that method of authentication. + +For that to work, the client needs to keep a cache of the web, keeping information about the tree hierarchy structure of +the URL space. + +To illustrate: + +1. a client goes to `https://alice.name/blog/2023/04/01/party.html` and receives a `401` with + a `Link: ; rel=effectiveAccessControl` header. +2. it fetches the `` resource to learn how to interact with `party.html`. +3. it fetches `/blog/2019/12/12/promise.html` with the right signature successfully + +If it later finds a link to `https://alice.name/blog/2019/12/12/promise.html` +then the client should try to immediately to sign on. + +But that means that the client needs to know that `https://alice.name/blog/2019/12/12/promise.html` is a resource that +is also dependent on the same `https://alice.name/blog/.acr`. How does it know that? Well, it can't **know** that +without getting a `401` from `promise.html` above. +But it can argue that `promise.html` is part of the container `https://alice.name/blog/` and that since that containers' +access control resource is `/blog/.acr` the same default set of rules should apply. But we have not actually yet got the +piece of info that the access control rule of `/blog/` is `/blog/.acr`. At least not in the right place namely in +the cache position `/blog/`. + +This could be solved in a number of ways. + +1. the "effectiveAccessControl" Link could instead of pointing to the ACR, point to the container, which would require + the client before 2. above to first do a `HEAD` on `/blog/` in order to find `/blog/.acr` +2. we could add to `/blog/.acr` a header `Link: <.>; rev="acl"; anchor=""` +3. the fact that `/blog/.acr` had a default rule on `/blog/` that enabled previous authentication should be good enough + +Only 1 gives us direct evidence which would be visible from the cache. +We could nevertheless in the Data structure for `/blog/` make space for incoming links. +That would allow us to add information at `/blog/` about the acl for that resource. \ No newline at end of file From 8764e5a79ee787fe603456f2558699451050e44c Mon Sep 17 00:00:00 2001 From: Henry Story Date: Wed, 10 May 2023 18:19:26 +0200 Subject: [PATCH 13/42] more notes after looking at mules lib --- .../src/main/scala/net/bblfish/web/README.md | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/wallet/shared/src/main/scala/net/bblfish/web/README.md b/wallet/shared/src/main/scala/net/bblfish/web/README.md index af7cb0d..f950356 100644 --- a/wallet/shared/src/main/scala/net/bblfish/web/README.md +++ b/wallet/shared/src/main/scala/net/bblfish/web/README.md @@ -7,7 +7,7 @@ We need a cache for web resources. We could define a Cache for a given X and Y something like ```scala -type Cache = Map[(Protocol, Domain, port), DirTree[X, Y]] +type Cache[X] = Map[(Protocol, Domain, port), DirTree[X]] ``` Arguably the domain and subdomains could also be part of the dirtree, @@ -15,7 +15,7 @@ if one wanted to think of domains hierarchically. Some differences: -1. dirtree.insert removes subtrees when adding a ref for a container. +1. dirtree.insert removes subtrees when adding an ActorRef for a container. But: * would placing the cached content in the attribute help? * could one just rename this as `insertAndRemoveSubDirs` and create a @@ -135,14 +135,44 @@ light and by the rules of politeness. ### Using actors as a Tree That is actually what Akka does. Each actor has children in the form of a tree. The only reason we needed our -`DirTree` immutable data structure there was to more efficiently find references for those actors so that we did not +`DirTree` immutable data structure in Reactive-Solid was to more efficiently find references for those actors so that we did not have to have * all messages pass through one root actor and * so we could reduce the message passing from paths of length n to 1 But the major advantage of actors is that they make it easy to check for files only when they are needed. So we don't have to for example update the whole map by reading all the files present in one directory. We can just check that -the directory or file we need is there. (see question 1 in [using Cofree](#using-catsfreecomonadfx) above) +the directory or file we need is there. (see question 1 in [using Cofree](#using-catsfreecomonadfx) above). + +Note: The gist [Typed Actors using Cats Effect, FS2 and Deferred Magic](https://gist.github.com/Swoorup/1ac9b69e0c0f1c0925d1397a94b0a762) could be useful. + +### Abstracting in the manner of Mules + +[Mules](https://github.com/davenverse/mules) is a Scala cache library that has +an http implementation [mules-http4s](https://github.com/davenverse/mules-http4s/tree/main). +It uses the Tagless Final pattern to define [Cache Traits](https://github.com/davenverse/mules/blob/main/modules/core/src/main/scala/io/chrisdavenport/mules/Cache.scala) +such as + +```scala +trait Get[F[_], K, V]{ + def get(k: K): F[V] +} +trait Lookup[F[_], K, V]{ + def lookup(k: K): F[Option[V]] +} +``` +which are then brought together in a Cache +```scala +trait Cache[F[_], K, V] + extends Lookup[F, K, V] + with Insert[F, K, V] + with Delete[F, K] +``` +But for our purposes it needs also a basic Search since we want to for example for a given +URI find out if one of the subdirectories that contain [the default ACL](#finding-the-default-acl). +That means that we need to have structure on the key. Would having an ordering of keys so that +one can find the closest smallest key (urlpath) to a given requested one? + ## Finding the default ACL From e2e665805a9062c3beaf6fa8ae0d4a107249d0fa Mon Sep 17 00:00:00 2001 From: Henry Story Date: Thu, 11 May 2023 17:27:32 +0200 Subject: [PATCH 14/42] update with final tagless idea from mules --- .../src/main/scala/net/bblfish/web/README.md | 406 +++++++++++++----- 1 file changed, 299 insertions(+), 107 deletions(-) diff --git a/wallet/shared/src/main/scala/net/bblfish/web/README.md b/wallet/shared/src/main/scala/net/bblfish/web/README.md index f950356..0993d86 100644 --- a/wallet/shared/src/main/scala/net/bblfish/web/README.md +++ b/wallet/shared/src/main/scala/net/bblfish/web/README.md @@ -1,15 +1,268 @@ ## The Cache -We need a cache for web resources. +We need a cache for web resources that can be used by the Wallet -### Use ReactiveSolid's DirTree? +## Use Cases -We could define a Cache for a given X and Y something like +Apart from the very much needed use case of speeding up clients, reducing traffic +and perhaps even keeping an archive of what was found. + +### Finding the default ACL + +One major use case for our cache which is a bit unusual from other use cases, is that we +want a cache for +the wallet to find the defaul ACL. + +The client needs a cache it can use to guess for as many resources as possible what the +default acl for them could be. +without going through a `401` response. After all, if a client has fetched a resource in a +container and later +wants to fetch other resources in that container or subcontainer then if the rule it used +to authenticate was a default +rule then it makes sense that it could continue using that method of authentication. + +For that to work, the client needs to keep a cache of the web, keeping information about +the tree hierarchy structure of +the URL space. + +To illustrate: + +1. a client goes to `https://alice.name/blog/2023/04/01/party.html` and receives a `401` + with + a `Link: ; rel=effectiveAccessControl` header. +2. it fetches the `` resource to learn how to interact with `party.html`. +3. it fetches `/blog/2019/12/12/promise.html` with the right signature successfully + +If it later finds a link to `https://alice.name/blog/2019/12/12/promise.html` +then the client should try to immediately to sign on. + +But that means that the client needs to know +that `https://alice.name/blog/2019/12/12/promise.html` is a resource that +is also dependent on the same `https://alice.name/blog/.acr`. How does it know that? Well, +it can't **know** that +without getting a `401` from `promise.html` above. +But it can argue that `promise.html` is part of the container `https://alice.name/blog/` +and that since that containers' +access control resource is `/blog/.acr` the same default set of rules should apply. But we +have not actually yet got the +piece of info that the access control rule of `/blog/` is `/blog/.acr`. At least not in +the right place namely in +the cache position `/blog/`. + +This could be solved in a number of ways. + +1. the "effectiveAccessControl" Link could instead of pointing to the ACR, point to the + container, which would require + the client before 2. above to first do a `HEAD` on `/blog/` in order to + find `/blog/.acr` +2. we could add to `/blog/.acr` a header `Link: <.>; rev="acl"; anchor=""` +3. the fact that `/blog/.acr` had a default rule on `/blog/` that enabled previous + authentication should be good enough + +Only 1 gives us direct evidence which would be visible from the cache. +We could nevertheless in the Data structure for `/blog/` make space for incoming links. +That would allow us to add information at `/blog/` about the acl for that resource. + +## Data Strutures + +There are 2 ways of modelling a web cache + +1. using a key-value store +2. keeping the Url tree hierarchy information + +### The Key/Value store + +The Key/Value store boils down to some simple structure such as + +```scala +type KVCache[X] = Map[Uri, X] +``` + +with ways of updating the Map with new content. +If we want to find the default ACL for a resource we would need then to do multiple +requests +to the `KVCache` going down one directory at a time on the cache. So if the client +wanted to find the ACL for `https://alice.name/events/2023/04/01/birthday` it would have +to go to + +1. `https://alice.name/events/2023/04/01/birthday` +2. `https://alice.name/events/2023/04/01/` +3. `https://alice.name/events/2023/04/` + +Until one found some content with a header pointing to the default acl. + +Notes: + +1. We will probably have most requests return None. + +### The Directory Tree + +A directory path with content at the nodes is a recursive +data structure which we can call DirTree, which is approximately + +```scala +case class DirTree[R](ref: R, kids: Map[String, DirTree[R]]) +``` + +That structure allows us to represent directory trees. But +we want one for each web server. So we need to extend it with +the Authority information which we can represent by the +triple`(Protocol, Domain, Port)` so that we have: ```scala type Cache[X] = Map[(Protocol, Domain, port), DirTree[X]] ``` +There are quite a few variation on representing the DirTree as +we will see. + +A `DirTree` for a cache is likely to have most nodes be unknown, so +that we will have `X = Option[Y]`. + +Advantages: + +* maps well to file system as a way of storing the data +* can find default acls as we walk up the hierarchy + +### Comparison between Key/Value and DirTree? + +At first sight one seems to be able to do the same in both. +But there are important differences. + +#### KVCache is low resolution + +With a KV datastore one could make a large number of requests to a server without +ever realising that one has no information at all about it. So if one had a url with 40 +slashes, one may need to make 40 requests to the KV DB before concluding that one had no +information about the default acl. + +With the Tree one would know immediately, just from the fact that one could +not find the server. If one had the server and a few directories, one would +find out by walking the tree hierarchy what the first container one knew about was. + +So if one finds one knows nothing about `https://alice.name/events/` and we don't +have a default acl even for that then we can immediately +fetch `https://alice.name/events/2023/04/01/birthday` on the Web, it should return a 401 +with a Link header. + +#### Better mapping to directory structure + +A DirTree view is going to be closer to the directory structure in which one can save +representations, which makes it easier to the find documents using normal unix commands. + +That means that representations using relative URL could function on the +local file system correctly, allowing one to view files locally, as rendered. +(Note there will be subtle problems with naming) + +It also means we could use the actor hierarchy for working with the caches too. + +## Using Key-Value DBs + +[Mules](https://github.com/davenverse/mules) is a Scala library that abstracts over +key-value pairs using the Tagless final pattern. +Here the DB is something simple like + +```scala +type Cache[X] = Map[Uri, X] +``` + +### Abstracting in the manner of Mules + +An http +implementation [mules-http4s](https://github.com/davenverse/mules-http4s/tree/main). +It uses the Tagless Final pattern to +define [Cache Traits](https://github.com/davenverse/mules/blob/main/modules/core/src/main/scala/io/chrisdavenport/mules/Cache.scala) +such as + +```scala +trait Get[F[_], K, V] { + def get(k: K): F[V] +} + +trait Lookup[F[_], K, V] { + def lookup(k: K): F[Option[V]] +} + +trait Insert[F[_], K, V] { + def insert(k: K, v: V): F[Unit] +} +``` + +which are then brought together in a Cache + +```scala +trait Cache[F[_], K, V] + extends Lookup[F, K, V] with Insert[F, K, V] with Delete[F, K] +``` + +Could we extend that with a basic `Search` trait? +We want to for a given URI find out if one of the subdirectories +contains [the default ACL](#finding-the-default-acl). + +```scala +trait Search[F[_], K, V] { + def find(start: K)(f: (K, V) => Either[K, Option[V]]): F[Option[V]] +} +``` + +Essentially this could run the function evaluated on the cache if possible to speed +things up. This function would start by fetching the original url, and pass the pair of +that url and the value to the function `f` and return either a new key to search on the +next iteration or the resulting value to end the calculation. + +This would allow one to search Key/Value store and follow links without the store +organising its knowledge using the tree hierarchy. Though it could. + +The `Search` trait could also be implemented externally using the previous Cache +functions. + +#### Questions: + +1. Could a `Cache` including the `Search` trait provide the abstraction needed to cover + both KeyValue DBs like [Caffeine](https://github.com/ben-manes/caffeine) and `TreeDir` + like databases? +2. Could such a `Cache` trait also be used to create Free Monads of Commands that could be + sent to our actor based cache? + +If 1 were true then 2 could be built out of the other methods without needing `Search`. + +The answer to 2 is yes. This is clearly explained in the Smithy4S +documentation [The Duality of Final and Initial algebras](https://disneystreaming.github.io/smithy4s/docs/design/services/#the-duality-of-final-and-initial-algebras) +where they mean the duality between finally encoded and initially encoded algebras, ie +between Free Monads and finally Tagless. And even better here is that the example given is +of a `KVStore`! + +Not only that but it suggests that we could use the Finally Tagless model for +HTTP requests, and then have an interpreter for the Resource, Container, ACL +etc, with implementations on how to write to file, create new dir, etc... + +Then our natural transformation interpreter would just interpret the +incoming messages. We should be careful to return results in such a +way that the actor can from the result decide how to send it on. + +## Implementations of DirTree + +### Using DirTree from ReactiveSolid + +In ReactiveSolid we implemented an +immutable [DirTree](https://github.com/co-operating-systems/Reactive-SoLiD/blob/30d6fd46fe30bc8be350c4a09d8592b20b791aa7/src/main/scala/run/cosy/ldp/DirTree.scala#L135). +Simplifiying just a little it is defined as: + +```scala +case class DirTree[R](ref: R, kids: scala.collection.immutable.HashMap[String, DirTree[R]]) +``` + +The library then implements its own zipper class to allow insertions and +changes to that immutable recursive data structure. + +Because it is immutable: + +* read access is easy +* write access will change the whole tree including the root. + +If a client wants access to the latest version, we need to wrap the root in a +java `AtomicReference[DirTree[R, A]]` or a cats effect `Ref` + Arguably the domain and subdomains could also be part of the dirtree, if one wanted to think of domains hierarchically. @@ -32,15 +285,12 @@ Some differences: Improvements needed: 1. Metadata: the container points to its ACL, - but as explained [below](#finding-the-default-acl) we may not have done a `GET` onthe container of the effective acl, - so when we work through a path hierarchy we would not have the info there to know that the container had a default + but as explained [below](#finding-the-default-acl) we may not have done a `GET` onthe + container of the effective acl, + so when we work through a path hierarchy we would not have the info there to know that + the container had a default acl. A good answer would be to allow one to add incoming links to the data structure. -## Implementations - -There are a number of ways of modelling the file structure of a server which is essentially a Tree with data at the -nodes. - ### Old OO style In old OO style we could just use mutable objects. @@ -57,27 +307,6 @@ changing something in the tree would just require That requires the whole tree to be synchronised for writes if paralle access is possible as it is in Java. -### Using DirTree from ReactiveSolid - -In ReactiveSolid we implemented an -immutable [DirTree](https://github.com/co-operating-systems/Reactive-SoLiD/blob/30d6fd46fe30bc8be350c4a09d8592b20b791aa7/src/main/scala/run/cosy/ldp/DirTree.scala#L135). -Simplifiying just a little it is defined as: - -```scala -case class DirTree[R](ref: R, kids: scala.collection.immutable.HashMap[String, DirTree[R]]) -``` - -The library then implements its own zipper class to allow insertions and -changes to that immutable recursive data structure. - -Because it is immutable: - -* read access is easy -* write access will change the whole tree including the root. - -If a client wants access to the latest version, we need to wrap the root in a -java `AtomicReference[DirTree[R, A]]` or a cats effect `Ref` - ### Using `cats.free.Comonad[F[_],X]` The definition of `cats.free.Cofree` is @@ -110,18 +339,24 @@ Questions: 1. could the lazy evaluation be useful with `Eval.Later` or `Eval.Always`? * when loading data from the file system? - Not really because if all we want is `/foo/bar/baz` we will need to load the Maps for `/foo` and `/foo/bar`, and - since we are dealing with immutable data structures we have to load all the elements of the map at once. This - actually shows the advantages of our tree of Actors, as those can only load one relation of the map when requested + Not really because if all we want is `/foo/bar/baz` we will need to load the Maps + for `/foo` and `/foo/bar`, and + since we are dealing with immutable data structures we have to load all the elements + of the map at once. This + actually shows the advantages of our tree of Actors, as those can only load one + relation of the map when requested by checking the file system. ### Using `cats.effect.kernel.Ref` -In a strongly concurrent system with many threads and potentially a lot of writes, we may want to model +In a strongly concurrent system with many threads and potentially a lot of writes, we may +want to model the Tree as an immutable structure using [cats.effect.kernel.Ref](https://typelevel.org/cats-effect/api/3.x/cats/effect/kernel/Ref.html). -That starts to be a lot closer to the original OO method of programming, as we no longer need a zipper to change -a node, but can just change the node itself, by for example creating a new `kids` in the parent, and replacing the +That starts to be a lot closer to the original OO method of programming, as we no longer +need a zipper to change +a node, but can just change the node itself, by for example creating a new `kids` in the +parent, and replacing the old one by updating the Ref. ```scala @@ -134,81 +369,38 @@ light and by the rules of politeness. ### Using actors as a Tree -That is actually what Akka does. Each actor has children in the form of a tree. The only reason we needed our -`DirTree` immutable data structure in Reactive-Solid was to more efficiently find references for those actors so that we did not -have to have -* all messages pass through one root actor and -* so we could reduce the message passing from paths of length n to 1 - -But the major advantage of actors is that they make it easy to check for files only when they are needed. So we don't -have to for example update the whole map by reading all the files present in one directory. We can just check that -the directory or file we need is there. (see question 1 in [using Cofree](#using-catsfreecomonadfx) above). - -Note: The gist [Typed Actors using Cats Effect, FS2 and Deferred Magic](https://gist.github.com/Swoorup/1ac9b69e0c0f1c0925d1397a94b0a762) could be useful. - -### Abstracting in the manner of Mules - -[Mules](https://github.com/davenverse/mules) is a Scala cache library that has -an http implementation [mules-http4s](https://github.com/davenverse/mules-http4s/tree/main). -It uses the Tagless Final pattern to define [Cache Traits](https://github.com/davenverse/mules/blob/main/modules/core/src/main/scala/io/chrisdavenport/mules/Cache.scala) -such as +That is actually what Akka does. Each actor has children in the form of a tree. The only +reason we needed our +`DirTree` immutable data structure in Reactive-Solid was to more efficiently find +references for those actors so that we +did not +have to have -```scala -trait Get[F[_], K, V]{ - def get(k: K): F[V] -} -trait Lookup[F[_], K, V]{ - def lookup(k: K): F[Option[V]] -} -``` -which are then brought together in a Cache -```scala -trait Cache[F[_], K, V] - extends Lookup[F, K, V] - with Insert[F, K, V] - with Delete[F, K] -``` -But for our purposes it needs also a basic Search since we want to for example for a given -URI find out if one of the subdirectories that contain [the default ACL](#finding-the-default-acl). -That means that we need to have structure on the key. Would having an ordering of keys so that -one can find the closest smallest key (urlpath) to a given requested one? +* all messages pass through one root actor and +* so we could reduce the message passing from paths of length n to 1 +But the major advantage of actors is that they make it easy to check for files only when +they are needed. So we don't +have to for example update the whole map by reading all the files present in one +directory. We can just check that +the directory or file we need is there. (see question 1 +in [using Cofree](#using-catsfreecomonadfx) above). -## Finding the default ACL +Note: The +gist [Typed Actors using Cats Effect, FS2 and Deferred Magic](https://gist.github.com/Swoorup/1ac9b69e0c0f1c0925d1397a94b0a762) +could be useful. -The client needs a cache it can use to guess for as many resources as possible what the default acl for them could be. -without going through a `401` response. After all, if a client has fetched a resource in a container and later -wants to fetch other resources in that container or subcontainer then if the rule it used to authenticate was a default -rule then it makes sense that it could continue using that method of authentication. +## Conclusions -For that to work, the client needs to keep a cache of the web, keeping information about the tree hierarchy structure of -the URL space. +The complications of implementations can be hidden behind a final tagless interface as is +done with [Mules](https://github.com/davenverse/mules). This allows us to switch +between implementations as suite the environment, yet write the Wallet +in a simple way. -To illustrate: +* Can one use Free Monads with actor interpretaers for our cache? + - yes +* Can one use the `Finally Tagless` method to abstract a Free Monad encoding that + would also work with sending messages to actors using say a much simpler interpreter + - yes. Finally tagless can be converted to Free Monads without loss -1. a client goes to `https://alice.name/blog/2023/04/01/party.html` and receives a `401` with - a `Link: ; rel=effectiveAccessControl` header. -2. it fetches the `` resource to learn how to interact with `party.html`. -3. it fetches `/blog/2019/12/12/promise.html` with the right signature successfully -If it later finds a link to `https://alice.name/blog/2019/12/12/promise.html` -then the client should try to immediately to sign on. - -But that means that the client needs to know that `https://alice.name/blog/2019/12/12/promise.html` is a resource that -is also dependent on the same `https://alice.name/blog/.acr`. How does it know that? Well, it can't **know** that -without getting a `401` from `promise.html` above. -But it can argue that `promise.html` is part of the container `https://alice.name/blog/` and that since that containers' -access control resource is `/blog/.acr` the same default set of rules should apply. But we have not actually yet got the -piece of info that the access control rule of `/blog/` is `/blog/.acr`. At least not in the right place namely in -the cache position `/blog/`. - -This could be solved in a number of ways. - -1. the "effectiveAccessControl" Link could instead of pointing to the ACR, point to the container, which would require - the client before 2. above to first do a `HEAD` on `/blog/` in order to find `/blog/.acr` -2. we could add to `/blog/.acr` a header `Link: <.>; rev="acl"; anchor=""` -3. the fact that `/blog/.acr` had a default rule on `/blog/` that enabled previous authentication should be good enough - -Only 1 gives us direct evidence which would be visible from the cache. -We could nevertheless in the Data structure for `/blog/` make space for incoming links. -That would allow us to add information at `/blog/` about the acl for that resource. \ No newline at end of file From 7218707b3ad13e9e3d7f3e30065e5931629da35b Mon Sep 17 00:00:00 2001 From: Henry Story Date: Thu, 11 May 2023 18:21:33 +0200 Subject: [PATCH 15/42] add an advantage to DirTree mapping. --- wallet/shared/src/main/scala/net/bblfish/web/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wallet/shared/src/main/scala/net/bblfish/web/README.md b/wallet/shared/src/main/scala/net/bblfish/web/README.md index 0993d86..7c22e80 100644 --- a/wallet/shared/src/main/scala/net/bblfish/web/README.md +++ b/wallet/shared/src/main/scala/net/bblfish/web/README.md @@ -156,6 +156,10 @@ local file system correctly, allowing one to view files locally, as rendered. It also means we could use the actor hierarchy for working with the caches too. +The advantage is also that as the search walks through the tree hierarchy it will find +the closest default ACL it knows about. The further it gets to the resource the more +precise it's knowledge of the default acl will become (if it has one). + ## Using Key-Value DBs [Mules](https://github.com/davenverse/mules) is a Scala library that abstracts over From 2225b20fea12ea647038a482e29cbdb83364e12b Mon Sep 17 00:00:00 2001 From: Henry Story Date: Thu, 11 May 2023 18:43:53 +0200 Subject: [PATCH 16/42] link to cache rfc --- wallet/shared/src/main/scala/net/bblfish/web/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wallet/shared/src/main/scala/net/bblfish/web/README.md b/wallet/shared/src/main/scala/net/bblfish/web/README.md index 7c22e80..bc21e90 100644 --- a/wallet/shared/src/main/scala/net/bblfish/web/README.md +++ b/wallet/shared/src/main/scala/net/bblfish/web/README.md @@ -1,6 +1,7 @@ ## The Cache -We need a cache for web resources that can be used by the Wallet +We need a cache for web resources that can be used by the Wallet. +Ideally it would also be a good tool for implementing [RFC 911: HTTP Caching](https://httpwg.org/specs/rfc9111.html#caching.negotiated.responses). ## Use Cases From ce79515d1c2b2ed0d1dc5239ab6cfee771115394 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Sun, 14 May 2023 21:21:55 +0200 Subject: [PATCH 17/42] add simple tree based cache lib --- build.sbt | 40 +++++++ .../scala/run/cosy/http/cache/Cache.scala | 57 ++++++++++ .../scala/run/cosy/http/cache/DirTree.scala | 104 ++++++++++++++++++ .../scala/run/cosy/http/cache/CacheTest.scala | 65 +++++++++++ .../run/cosy/http/cache/DirTreeTest.scala | 104 ++++++++++++++++++ project/Dependencies.scala | 9 ++ .../net/bblfish/wallet/BasicAuthWallet.scala | 7 ++ .../src/main/scala/net/bblfish/web/README.md | 3 +- 8 files changed, 388 insertions(+), 1 deletion(-) create mode 100644 cache/shared/src/main/scala/run/cosy/http/cache/Cache.scala create mode 100644 cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala create mode 100644 cache/shared/src/test/scala/run/cosy/http/cache/CacheTest.scala create mode 100644 cache/shared/src/test/scala/run/cosy/http/cache/DirTreeTest.scala diff --git a/build.sbt b/build.sbt index f73833e..c68cd18 100644 --- a/build.sbt +++ b/build.sbt @@ -123,6 +123,45 @@ lazy val ldes = crossProject(JVMPlatform) ) ) +// make a new project called cache +lazy val cache = crossProject(JVMPlatform) + .crossType(CrossType.Full) + .in(file("cache")) + .settings(commonSettings: _*) + .settings( + name := "Cache", + description := "Cache", + libraryDependencies ++= Seq( + cats.core.value, + cats.free.value, + http4s.core.value, + mules.core.value + ), + libraryDependencies ++= Seq( +// munit.value % Test, + cats.munitEffect.value % Test, + ) + ) + + +lazy val test = crossProject(JVMPlatform) + .crossType(CrossType.Full) + .in(file("test")) + .settings(commonSettings: _*) + .settings( + name := "test", + description := "test stuff", + libraryDependencies ++= Seq( + mules.http4s.value, + mules.core.value, + mules.caffeine.value, + mules.ember_client.value + ), + libraryDependencies ++= Seq( + munit.value % Test + ) + ) + // todo: should be moved closer to banana-rdf repo lazy val ioExt4s = crossProject(JVMPlatform) .crossType(CrossType.Full) @@ -157,6 +196,7 @@ lazy val wallet = crossProject(JVMPlatform) // , JSPlatform) resolvers += sonatypeSNAPSHOT, libraryDependencies ++= Seq( http4s.client.value, + cats.free.value, http4s.ember_client.value, // <- remove. added to explore implementation banana.bananaJena.value, crypto.http4sSig.value diff --git a/cache/shared/src/main/scala/run/cosy/http/cache/Cache.scala b/cache/shared/src/main/scala/run/cosy/http/cache/Cache.scala new file mode 100644 index 0000000..d2b0880 --- /dev/null +++ b/cache/shared/src/main/scala/run/cosy/http/cache/Cache.scala @@ -0,0 +1,57 @@ +package run.cosy.http.cache + +import cats.effect.kernel.{Ref, Sync} +import cats.syntax.all.* +import cats.{FlatMap, MonadError} +import io.chrisdavenport.mules.Cache +import org.http4s.Uri +import run.cosy.http.cache.DirTree.* +import run.cosy.http.cache.TreeDirCache.WebCache + +object TreeDirCache: + type Server = (Uri.Scheme, Uri.Authority) + type WebCache[X] = Map[Server, DirTree[Option[X]]] + +sealed trait TreeDirException extends Exception +case class ServerNotFound(uri: Uri) extends TreeDirException +case class IncompleteServiceInfo(uri: Uri) extends TreeDirException + +/** this is a mules cache, but we add a method to search the path */ +case class TreeDirCache[F[_], X]( + cacheRef: Ref[F, WebCache[X]] +)(using F: Sync[F]) + extends Cache[F, Uri, X]: + + override def delete(k: Uri): F[Unit] = ??? + + override def insert(k: Uri, v: X): F[Unit] = + for + scheme <- F.fromOption(k.scheme, IncompleteServiceInfo(k)) + auth <- F.fromOption(k.authority, IncompleteServiceInfo(k)) + server = (scheme, auth) + _ <- cacheRef.update { map => + val tree = map.getOrElse(server, DirTree.pure(None)) + map.updated(server, tree.insertAt(k.path.segments, Some(v), None)) + } + yield () + + override def lookup(k: Uri): F[Option[X]] = + for + scheme <- F.fromOption(k.scheme, IncompleteServiceInfo(k)) + auth <- F.fromOption(k.authority, IncompleteServiceInfo(k)) + webCache <- cacheRef.get + server = (scheme, auth) + tree <- F.fromOption(webCache.get(server), ServerNotFound(k)) + yield + val (path, v) = tree.find(k.path.segments) + if path.isEmpty then v else None + + def findClosest(k: Uri)(matcher: Option[X] => Boolean): F[Option[X]] = + for + scheme <- F.fromOption(k.scheme, IncompleteServiceInfo(k)) + auth <- F.fromOption(k.authority, IncompleteServiceInfo(k)) + webCache <- cacheRef.get + server = (scheme, auth) + tree <- F.fromOption(webCache.get(server), ServerNotFound(k)) + yield + tree.findClosest(k.path.segments)(matcher).flatten \ No newline at end of file diff --git a/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala b/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala new file mode 100644 index 0000000..022a6ab --- /dev/null +++ b/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala @@ -0,0 +1,104 @@ +package run.cosy.http.cache + +import cats.free.Cofree +import cats.Eval +import org.http4s.Uri.Path + +import scala.annotation.tailrec +import scala.util.Right + +object DirTree: + //todo: we really need a Uri abstraction + type Dir[X] = Map[org.http4s.Uri.Path.Segment, X] + type DirTree[X] = Cofree[Dir[_], X] + type Path = Seq[Path.Segment] + + /** A Path of DirTree[R,A]s is used to take apart a DirTree[R,A] structure, in the reverse + * direction. Note the idea is that each link (name, dt) points from dt via name in the hashMap + * to the next deeper node. The APath is in reverse direction so we have List( "newName" <- dir2 + * , "dir2in1" <- dir1 , "dir1" <- rootDir ) An empty List would just refer to the root + * DirTree[R,A] + */ + type ZPath[A] = Seq[ZLink[A]] + + /** A Zip Link + * @param from + * the DirTree from which the link is pointing + * @param linkName + * the name of the link. There may be nothing at the end of that. + */ + case class ZLink[A](from: DirTree[A], linkName: Path.Segment) + + /** The context of an unzipped DirTree the first projection is either + * -Right: the object at the end of the path + * -Left: the remaining path. If it is Nil then the path is pointing into a position to which one + * can add the second is the A Path, from the object to the root so that the object can be + * reconstituted + */ + type ZipContext[A] = (Either[Path, DirTree[A]], ZPath[A]) + + def pure[X](x: X): DirTree[X] = Cofree(x, Eval.now(Map.empty)) + def apply[X](x: X, dirs: Dir[DirTree[X]]): DirTree[X] = Cofree(x, Eval.now(dirs)) + + extension [X](thizDt: DirTree[X]) + def ->(name: Path.Segment): ZLink[X] = ZLink(thizDt, name) + + /** find the closest node X available when following Path + * return the remaining path */ + @tailrec + def find(at: Path): (Path, X) = + at match + case Seq() => (at, thizDt.head) + case Seq(name, tail*) => + val dir: Dir[Cofree[Dir, X]] = thizDt.tail.value + dir.get(name) match + case None => (at, thizDt.head) + case Some(tree) => tree.find(tail) + end find + + def unzipAlong(path: Path): ZipContext[X] = + @tailrec + def loop(dt: DirTree[X], path: Path, result: ZPath[X]): ZipContext[X] = + path match + case Seq() => (Right(dt), result) + case Seq(name, rest*) => + if dt.tail.value.isEmpty then (Left(rest), ZLink(dt, name) +: result) + else + dt.tail.value.get(name) match + case None => (Left(rest), ZLink(dt, name) +: result) + case Some(dtchild) => + loop(dtchild, rest, ZLink(dt, name) +: result) + end loop + + loop(thizDt, path, Seq()) + end unzipAlong + + /** find the closest node matching `select` going backwards from where we got */ + def findClosest(path: Path)(select: X => Boolean): Option[X] = + unzipAlong(path) match + case (Right(dt), zpath) => dt.head +: zpath.map(_.from.head) find select + case (Left(_), zpath) => zpath.map(_.from.head) find select + + /** Rezip along zpath */ + def rezip(path: ZPath[X]): DirTree[X] = + def loop(path: ZPath[X], dt: DirTree[X]): DirTree[X] = + path match + case Seq() => dt + case Seq(ZLink(from, name), tail*) => + val newTail = from.tail.value + (name -> dt) + loop(tail, from.copy(tail = Eval.now(newTail))) + loop(path, thizDt) + + /** set value at path creating new directories with default values if needed */ + def insertAt(path: Path, value: X, default: X): DirTree[X] = + thizDt.unzipAlong(path) match + case (Left(path), zpath) => + pure(value).rezip(path.reverse.map(p => pure(default) -> p).appendedAll(zpath)) + case (Right(dt), zpath) => dt.copy(head = value).rezip(zpath) + + /** set dirTree at path creating new directories with default values if needed + */ + def setDirAt(path: Path, dt: DirTree[X], default: X): DirTree[X] = + thizDt.unzipAlong(path) match + case (Left(path), zpath) => dt.rezip(path.map(p => pure(default) -> p).appendedAll(zpath)) + case (Right(dt), zpath) => dt.rezip(zpath) diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/CacheTest.scala b/cache/shared/src/test/scala/run/cosy/http/cache/CacheTest.scala new file mode 100644 index 0000000..1180061 --- /dev/null +++ b/cache/shared/src/test/scala/run/cosy/http/cache/CacheTest.scala @@ -0,0 +1,65 @@ +package run.cosy.http.cache + +import cats.MonadError +import cats.effect.{IO, Ref, SyncIO} +import munit.CatsEffectSuite +import org.http4s.Uri + +class CacheTest extends CatsEffectSuite: + import TreeDirCache.* + import cats.MonadError.* + import run.cosy.http.cache.DirTree + + def mkCache[X]: IO[TreeDirCache[IO, X]] = + for wc <- Ref.of[IO, WebCache[X]](Map.empty) + yield new TreeDirCache[IO, X](wc) + + val bbl = Uri.unsafeFromString("https://bblfish.net/people/henry/card#me") + val bblPplDir = Uri.unsafeFromString("https://bblfish.net/people/") + val bblRoot = Uri.unsafeFromString("https://bblfish.net/") + val anais = Uri.unsafeFromString("https://bblfish.net/people/anais/card#i") + + val cacheIO: IO[TreeDirCache[IO, Int]] = mkCache[Int] + + test("test url paths") { + assertEquals(bbl.path.segments.map(_.toString), Vector("people","henry","card")) + assertEquals(bblPplDir.path.segments.map(_.toString), Vector("people")) + } + + test("first test") { + for + cache <- cacheIO + x <- cache.insert(bbl, 3) + y <- cache.lookup(bbl) + yield + assertEquals(x, ()) + assertEquals(y, Some(3)) + } + + test("second test, does not capture the history of changes from the first.") { + val iofail: IO[Unit] = for + cache <- cacheIO + y <- cache.lookup(bbl) + yield () + interceptIO[ServerNotFound](iofail) + } + + test("test searching for a parent") { + for + cache <- cacheIO + _ <- cache.insert(bbl, 3) + _ <- cache.insert(bblPplDir,2) + _ <- cache.insert(bblRoot,0) + x <- cache.lookup(bblPplDir) + y <- cache.lookup(anais) + _ <- cache.insert(anais,12) + y2 <- cache.lookup(anais) + z <- cache.findClosest(bbl){ oi => oi == Some(0) } + w <- cache.findClosest(anais){ oi => oi == Some(2) } + yield + assertEquals(x, Some(2)) + assertEquals(y, None) + assertEquals(y2, Some(12)) + assertEquals(z, Some(0)) + assertEquals(w, Some(2)) + } diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/DirTreeTest.scala b/cache/shared/src/test/scala/run/cosy/http/cache/DirTreeTest.scala new file mode 100644 index 0000000..63da085 --- /dev/null +++ b/cache/shared/src/test/scala/run/cosy/http/cache/DirTreeTest.scala @@ -0,0 +1,104 @@ +package run.cosy.http.cache + +import scala.collection.immutable.HashMap +import DirTree.* +import org.http4s.Uri + +class DirTreeTest extends munit.FunSuite: + import scala.language.implicitConversions + implicit def s2p(s: String): Uri.Path.Segment = Uri.Path.Segment(s) + implicit def ps2p(ps: Seq[String]): Uri.Path = Uri.Path(ps.map(s2p).toVector) + def Pth(s: String*): Seq[Uri.Path.Segment] = s.map(s2p) + def Mp(sdt: (String, DT)*): Map[Uri.Path.Segment, DT] = + HashMap.from(sdt.map((s, dt) => Uri.Path.Segment(s) -> dt)) + type DT = DirTree[Int] + val srcTstScHello = Pth("src", "test", "scala", "hello.txt") + val fooBarBaz = Pth("foo", "bar", "baz") + val srcTst = srcTstScHello.take(2) + val src = srcTst.take(1) + val srcTsSc = srcTstScHello.take(3) + val one: DirTree[Int] = DirTree.pure[Int](1) + val twoOne = DirTree(2, Mp("test" -> one)) + val threeTwoOne = DirTree(3, Mp("src" -> twoOne)) + + import DirTree.* + val testScHello = srcTstScHello.drop(1) + + test("findClosest") { + assertEquals(one.find(Nil), (Nil, 1)) + assertEquals(one.find(fooBarBaz), (fooBarBaz, 1)) + assertEquals(one.find(srcTstScHello), (srcTstScHello, 1)) + + assertEquals(twoOne.find(Nil), (Nil, 2)) + assertEquals(twoOne.find(fooBarBaz), (fooBarBaz, 2)) + assertEquals(twoOne.find(testScHello), (testScHello.drop(1), 1)) + + assertEquals(threeTwoOne.find(Nil), (Nil, 3)) + assertEquals(threeTwoOne.find(fooBarBaz), (fooBarBaz, 3)) + assertEquals(threeTwoOne.find(srcTstScHello), (srcTstScHello.drop(2), 1)) + + } + + test("toClosestPath") { + assertEquals( + threeTwoOne.unzipAlong(Nil), + (Right(threeTwoOne), Nil) + ) + assertEquals( + threeTwoOne.unzipAlong(Pth("src")), + (Right(twoOne), Seq(ZLink(threeTwoOne, "src"))) + ) + assertEquals( + threeTwoOne.unzipAlong(Pth("src", "test")), + (Right(one), Seq(ZLink(twoOne, "test"), ZLink(threeTwoOne, "src"))) + ) + // note that "scala" here is not in One. It is pointing to a place we may want something to be. + assertEquals( + threeTwoOne.unzipAlong(Pth("src", "test", "scala")), + (Left(Nil), Seq(one -> "scala", twoOne -> "test", threeTwoOne -> "src")) + ) + assertEquals( + threeTwoOne.unzipAlong(Pth("src", "test", "scala", "hello.txt")), + (Left(Pth("hello.txt")), List(one -> "scala", twoOne -> "test", threeTwoOne -> "src")) + ) + } + + test("rezip") { + assertEquals(one.rezip(Nil), one) + assertEquals(one.rezip(Seq(DirTree.pure(2) -> "test")), twoOne) + assertEquals(one.rezip(Seq(DirTree.pure(2) -> "test", DirTree.pure(3) -> "src")), threeTwoOne) + } + + test("setAt") { + assertEquals(one.insertAt(Nil, 2, 0), DirTree.pure(2)) + assertEquals(one.insertAt(Nil, -1, 0), DirTree.pure(-1)) + assertEquals(one.insertAt(Seq("two"), 2, 0), DirTree(1, Mp("two" -> DirTree.pure(2)))) + assertEquals( + one.insertAt(Pth("two", "three"), 2, 0), + DirTree(1, Mp("two" -> DirTree(0, Mp("three" -> DirTree.pure(2))))) + ) + val tz1 = DirTree(3, Mp("zero" -> DirTree(0, Mp("one" -> DirTree.pure(1))))) + assertEquals(DirTree.pure(3).insertAt(Seq("zero", "one"), 1, 0), tz1) + val zom1t = DirTree( + 3, + Mp( + "zero" -> DirTree( + 0, + Mp("one" -> DirTree(1, Mp("minus1" -> DirTree(-1, Mp("two" -> DirTree.pure(2)))))) + ) + ) + ) + assertEquals(tz1.insertAt(Seq("zero", "one", "minus1", "two"), 2, -1), zom1t) + assertEquals( + zom1t.insertAt(Pth("zero", "one"), 55555, -2), + DirTree( + 3, + Mp( + "zero" -> DirTree( + 0, + Mp("one" -> DirTree(55555, Mp("minus1" -> DirTree(-1, Mp("two" -> DirTree.pure(2)))))) + ) + ) + ) + ) + } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index e642856..c2515e2 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -14,6 +14,15 @@ object Dependencies { // https://github.com/lemonlabsuk/scala-uri val scalaUri = Def.setting("io.lemonlabs" %%% "scala-uri" % "4.0.3") } + + object mules { + val core = Def.setting("io.chrisdavenport" %% "mules" % "0.7.0") + val caffeine = Def.setting("io.chrisdavenport" %% "mules-caffeine" % "0.7.0") + val http4s = Def.setting("io.chrisdavenport" %% "mules-http4s" % "0.4.0") + // ember uses 0.23.18 + val ember_client = Def.setting("org.http4s" %%% "http4s-ember-client" % "0.23.18") + } + // https://http4s.org/v1.0/client/ object http4s { def apply(packg: String): Def.Initialize[sbt.ModuleID] = diff --git a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala index e780ecd..8375df5 100644 --- a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala +++ b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala @@ -162,6 +162,13 @@ end WalletTools * given a DataSet proxied to the web to allow caching. On the other hand with free monads one * could have those be interpreted according to context... Todo: compare this way of working with * free-monads. + * + * @param db + * username/passwords per domain + * @param keyIdDB + * Key Database + * @param client + * The client needed to fetch acl resources on the web (should be a proxy) */ class BasicWallet[F[_], Rdf <: RDF]( db: Map[ll.Authority, BasicId], diff --git a/wallet/shared/src/main/scala/net/bblfish/web/README.md b/wallet/shared/src/main/scala/net/bblfish/web/README.md index bc21e90..0536c1b 100644 --- a/wallet/shared/src/main/scala/net/bblfish/web/README.md +++ b/wallet/shared/src/main/scala/net/bblfish/web/README.md @@ -1,7 +1,8 @@ ## The Cache We need a cache for web resources that can be used by the Wallet. -Ideally it would also be a good tool for implementing [RFC 911: HTTP Caching](https://httpwg.org/specs/rfc9111.html#caching.negotiated.responses). +Ideally it would also be a good tool for +implementing [RFC 911: HTTP Caching](https://httpwg.org/specs/rfc9111.html#caching.negotiated.responses). ## Use Cases From 5a0373fe5eda25e193d8ccf56f0276d8ad3f69b4 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Mon, 15 May 2023 10:58:45 +0200 Subject: [PATCH 18/42] add support for cache deletion --- .../scala/run/cosy/http/cache/Cache.scala | 49 ++++++++++++++++--- .../scala/run/cosy/http/cache/DirTree.scala | 30 +++++++++--- .../scala/run/cosy/http/cache/CacheTest.scala | 47 ++++++++++++------ 3 files changed, 96 insertions(+), 30 deletions(-) diff --git a/cache/shared/src/main/scala/run/cosy/http/cache/Cache.scala b/cache/shared/src/main/scala/run/cosy/http/cache/Cache.scala index d2b0880..c247b8a 100644 --- a/cache/shared/src/main/scala/run/cosy/http/cache/Cache.scala +++ b/cache/shared/src/main/scala/run/cosy/http/cache/Cache.scala @@ -11,7 +11,7 @@ import run.cosy.http.cache.TreeDirCache.WebCache object TreeDirCache: type Server = (Uri.Scheme, Uri.Authority) type WebCache[X] = Map[Server, DirTree[Option[X]]] - + sealed trait TreeDirException extends Exception case class ServerNotFound(uri: Uri) extends TreeDirException case class IncompleteServiceInfo(uri: Uri) extends TreeDirException @@ -22,16 +22,50 @@ case class TreeDirCache[F[_], X]( )(using F: Sync[F]) extends Cache[F, Uri, X]: - override def delete(k: Uri): F[Unit] = ??? + /* todo: consider if that is really wise. It doe */ + override def delete(k: Uri): F[Unit] = + if k.path.segments.isEmpty && !k.path.endsWithSlash + then F.pure(()) + else + for + scheme <- F.fromOption(k.scheme, IncompleteServiceInfo(k)) + auth <- F.fromOption(k.authority, IncompleteServiceInfo(k)) + server = (scheme, auth) + _ <- cacheRef.update { webCache => + webCache.get(server) match + case Some(tree) => webCache.updated(server, tree.set(k.path.segments, None)) + case None => webCache + } + yield () + + /** Deleting a server without a path, results in loosing all info in the path. todo: consider if + * that is really wise. It doe + */ + def deleteBelow(k: Uri): F[Unit] = + for + scheme <- F.fromOption(k.scheme, IncompleteServiceInfo(k)) + auth <- F.fromOption(k.authority, IncompleteServiceInfo(k)) + server = (scheme, auth) + _ <- cacheRef.update { webCache => + if k.path.segments.isEmpty && !k.path.endsWithSlash + then webCache - server + else + webCache + .get(server) + .map(_.setDirAt(k.path.segments, DirTree.pure(None))) match + case Some(tree) => webCache.updated(server, tree) + case None => webCache + } + yield () override def insert(k: Uri, v: X): F[Unit] = for scheme <- F.fromOption(k.scheme, IncompleteServiceInfo(k)) auth <- F.fromOption(k.authority, IncompleteServiceInfo(k)) server = (scheme, auth) - _ <- cacheRef.update { map => - val tree = map.getOrElse(server, DirTree.pure(None)) - map.updated(server, tree.insertAt(k.path.segments, Some(v), None)) + _ <- cacheRef.update { webCache => + val tree = webCache.getOrElse(server, DirTree.pure(None)) + webCache.updated(server, tree.insertAt(k.path.segments, Some(v), None)) } yield () @@ -45,7 +79,7 @@ case class TreeDirCache[F[_], X]( yield val (path, v) = tree.find(k.path.segments) if path.isEmpty then v else None - + def findClosest(k: Uri)(matcher: Option[X] => Boolean): F[Option[X]] = for scheme <- F.fromOption(k.scheme, IncompleteServiceInfo(k)) @@ -53,5 +87,4 @@ case class TreeDirCache[F[_], X]( webCache <- cacheRef.get server = (scheme, auth) tree <- F.fromOption(webCache.get(server), ServerNotFound(k)) - yield - tree.findClosest(k.path.segments)(matcher).flatten \ No newline at end of file + yield tree.findClosest(k.path.segments)(matcher).flatten diff --git a/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala b/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala index 022a6ab..3e401ed 100644 --- a/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala +++ b/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala @@ -1,14 +1,14 @@ package run.cosy.http.cache -import cats.free.Cofree import cats.Eval +import cats.free.Cofree import org.http4s.Uri.Path import scala.annotation.tailrec import scala.util.Right object DirTree: - //todo: we really need a Uri abstraction + // todo: we really need a Uri abstraction type Dir[X] = Map[org.http4s.Uri.Path.Segment, X] type DirTree[X] = Cofree[Dir[_], X] type Path = Seq[Path.Segment] @@ -43,8 +43,8 @@ object DirTree: extension [X](thizDt: DirTree[X]) def ->(name: Path.Segment): ZLink[X] = ZLink(thizDt, name) - /** find the closest node X available when following Path - * return the remaining path */ + /** find the closest node X available when following Path return the remaining path + */ @tailrec def find(at: Path): (Path, X) = at match @@ -72,8 +72,8 @@ object DirTree: loop(thizDt, path, Seq()) end unzipAlong - - /** find the closest node matching `select` going backwards from where we got */ + + /** find the closest node matching `select` going backwards from where we got */ def findClosest(path: Path)(select: X => Boolean): Option[X] = unzipAlong(path) match case (Right(dt), zpath) => dt.head +: zpath.map(_.from.head) find select @@ -96,9 +96,25 @@ object DirTree: pure(value).rezip(path.reverse.map(p => pure(default) -> p).appendedAll(zpath)) case (Right(dt), zpath) => dt.copy(head = value).rezip(zpath) + /** set value at point `path` but wihtout creating intermediary directories. This is actually + * useful for deleting an entry without affecting the environement: ie. only delete what exists + */ + def set(path: Path, value: X): DirTree[X] = + thizDt.unzipAlong(path) match + case (Right(dt), zpath) => dt.copy(head = value).rezip(zpath) + case _ => thizDt + + /** set value at point `path` but wihtout creating intermediary directories. This is actually + * useful for deleting an entry without affecting the environement: ie. only delete what exists + */ + def setDirAt(path: Path, newDt: DirTree[X]): DirTree[X] = + thizDt.unzipAlong(path) match + case (Right(_), zpath) => newDt.rezip(zpath) + case _ => thizDt + /** set dirTree at path creating new directories with default values if needed */ - def setDirAt(path: Path, dt: DirTree[X], default: X): DirTree[X] = + def insertDirAt(path: Path, dt: DirTree[X], default: X): DirTree[X] = thizDt.unzipAlong(path) match case (Left(path), zpath) => dt.rezip(path.map(p => pure(default) -> p).appendedAll(zpath)) case (Right(dt), zpath) => dt.rezip(zpath) diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/CacheTest.scala b/cache/shared/src/test/scala/run/cosy/http/cache/CacheTest.scala index 1180061..b9a4d6f 100644 --- a/cache/shared/src/test/scala/run/cosy/http/cache/CacheTest.scala +++ b/cache/shared/src/test/scala/run/cosy/http/cache/CacheTest.scala @@ -18,12 +18,12 @@ class CacheTest extends CatsEffectSuite: val bblPplDir = Uri.unsafeFromString("https://bblfish.net/people/") val bblRoot = Uri.unsafeFromString("https://bblfish.net/") val anais = Uri.unsafeFromString("https://bblfish.net/people/anais/card#i") - + val cacheIO: IO[TreeDirCache[IO, Int]] = mkCache[Int] - + test("test url paths") { - assertEquals(bbl.path.segments.map(_.toString), Vector("people","henry","card")) - assertEquals(bblPplDir.path.segments.map(_.toString), Vector("people")) + assertEquals(bbl.path.segments.map(_.toString), Vector("people", "henry", "card")) + assertEquals(bblPplDir.path.segments.map(_.toString), Vector("people")) } test("first test") { @@ -45,21 +45,38 @@ class CacheTest extends CatsEffectSuite: } test("test searching for a parent") { - for + val ioCache = for cache <- cacheIO _ <- cache.insert(bbl, 3) - _ <- cache.insert(bblPplDir,2) - _ <- cache.insert(bblRoot,0) + _ <- cache.insert(bblPplDir, 2) + _ <- cache.insert(bblRoot, 0) x <- cache.lookup(bblPplDir) y <- cache.lookup(anais) - _ <- cache.insert(anais,12) + _ <- cache.insert(anais, 12) y2 <- cache.lookup(anais) - z <- cache.findClosest(bbl){ oi => oi == Some(0) } - w <- cache.findClosest(anais){ oi => oi == Some(2) } + z <- cache.findClosest(bbl) { _ == Some(0) } + w <- cache.findClosest(anais) { _ == Some(2) } + yield + assertEquals(x, Some(2)) + assertEquals(y, None) + assertEquals(y2, Some(12)) + assertEquals(z, Some(0)) + assertEquals(w, Some(2)) + cache + + for + cache <- ioCache + a <- cache.lookup(anais) + _ <- cache.delete(bblPplDir) + x <- cache.lookup(bblPplDir) + a2 <- cache.lookup(anais) + _ <- cache.deleteBelow(bblPplDir) + a3 <- cache.lookup(anais) + y3 <- cache.lookup(bbl) yield - assertEquals(x, Some(2)) - assertEquals(y, None) - assertEquals(y2, Some(12)) - assertEquals(z, Some(0)) - assertEquals(w, Some(2)) + assertEquals(a, Some(12)) + assertEquals(x, None) + assertEquals(a2, Some(12)) + assertEquals(a3, None) + assertEquals(y3, None) } From 42a750142feeda2bcf029b02eae4db1f359399ab Mon Sep 17 00:00:00 2001 From: Henry Story Date: Mon, 15 May 2023 11:08:15 +0200 Subject: [PATCH 19/42] move scripts to shared space --- .../{jvm => shared}/src/main/scala/scripts/AnHttpSigClient.scala | 0 scripts/{jvm => shared}/src/main/scala/scripts/MiniCF.scala | 0 scripts/{jvm => shared}/src/main/scala/scripts/PemToJWT.sc | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename scripts/{jvm => shared}/src/main/scala/scripts/AnHttpSigClient.scala (100%) rename scripts/{jvm => shared}/src/main/scala/scripts/MiniCF.scala (100%) rename scripts/{jvm => shared}/src/main/scala/scripts/PemToJWT.sc (100%) diff --git a/scripts/jvm/src/main/scala/scripts/AnHttpSigClient.scala b/scripts/shared/src/main/scala/scripts/AnHttpSigClient.scala similarity index 100% rename from scripts/jvm/src/main/scala/scripts/AnHttpSigClient.scala rename to scripts/shared/src/main/scala/scripts/AnHttpSigClient.scala diff --git a/scripts/jvm/src/main/scala/scripts/MiniCF.scala b/scripts/shared/src/main/scala/scripts/MiniCF.scala similarity index 100% rename from scripts/jvm/src/main/scala/scripts/MiniCF.scala rename to scripts/shared/src/main/scala/scripts/MiniCF.scala diff --git a/scripts/jvm/src/main/scala/scripts/PemToJWT.sc b/scripts/shared/src/main/scala/scripts/PemToJWT.sc similarity index 100% rename from scripts/jvm/src/main/scala/scripts/PemToJWT.sc rename to scripts/shared/src/main/scala/scripts/PemToJWT.sc From b939fde884ebe42f6923c6de818f275691d22431 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Mon, 15 May 2023 18:04:53 +0200 Subject: [PATCH 20/42] copy of mules-http files by chrisdavenport --- .../mules/http4s/CacheItem.scala | 35 +++ .../mules/http4s/CachedResponse.scala | 48 ++++ .../mules/http4s/internal/CacheRules.scala | 236 ++++++++++++++++++ .../mules/http4s/internal/Caching.scala | 76 ++++++ 4 files changed, 395 insertions(+) create mode 100644 cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala create mode 100644 cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala create mode 100644 cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/CacheRules.scala create mode 100644 cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala new file mode 100644 index 0000000..426dd19 --- /dev/null +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala @@ -0,0 +1,35 @@ +package io.chrisdavenport.mules.http4s + +import cats._ +import cats.effect._ +import cats.implicits._ +import org.http4s.HttpDate + +/** + * Cache Items are what we place in the cache, this is exposed + * so that caches can be constructed by the user for this type + **/ +final case class CacheItem( + created: HttpDate, + expires: Option[HttpDate], + response: CachedResponse, +) + +object CacheItem { + + def create[F[_]: Clock: MonadThrow](response: CachedResponse, expires: Option[HttpDate]): F[CacheItem] = + HttpDate.current[F].map(date => + new CacheItem(date, expires, response) + ) + + private[http4s] final case class Age(val deltaSeconds: Long) extends AnyVal + private[http4s] object Age { + def of(created: HttpDate, now: HttpDate): Age = new Age(now.epochSecond - created.epochSecond) + } + private[http4s] final case class CacheLifetime(val deltaSeconds: Long) extends AnyVal + private[http4s] object CacheLifetime { + def of(expires: Option[HttpDate], now: HttpDate): Option[CacheLifetime] = expires.map{expiredAt => + new CacheLifetime(expiredAt.epochSecond - now.epochSecond) + } + } +} \ No newline at end of file diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala new file mode 100644 index 0000000..59f9132 --- /dev/null +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala @@ -0,0 +1,48 @@ +package io.chrisdavenport.mules.http4s + +import org.typelevel.vault.Vault +import org.http4s._ +import fs2._ + +import cats._ +import cats.implicits._ +import scodec.bits.ByteVector + +// As attributes can be unbound. We cannot cache them as they may not be safe to do so. +final case class CachedResponse( + status: Status, + httpVersion: HttpVersion, + headers: Headers, + body: ByteVector +){ + def withHeaders(headers: Headers): CachedResponse = new CachedResponse( + this.status, + this.httpVersion, + headers, + this.body + ) + def toResponse[F[_]]: Response[F] = CachedResponse.toResponse(this) +} + +object CachedResponse { + + def fromResponse[F[_], G[_]: Functor](response: Response[F])(implicit compiler: Compiler[F,G]): G[CachedResponse] = { + response.body.compile.to(ByteVector).map{bv => + new CachedResponse( + response.status, + response.httpVersion, + response.headers, + bv + ) + } + } + + def toResponse[F[_]](cachedResponse: CachedResponse): Response[F] = + Response( + cachedResponse.status, + cachedResponse.httpVersion, + cachedResponse.headers, + Stream.chunk(Chunk.byteVector(cachedResponse.body)), + Vault.empty + ) +} \ No newline at end of file diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/CacheRules.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/CacheRules.scala new file mode 100644 index 0000000..5e5191f --- /dev/null +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/CacheRules.scala @@ -0,0 +1,236 @@ +package io.chrisdavenport.mules.http4s.internal + +import io.chrisdavenport.mules.http4s._ + +import org.http4s._ +import org.http4s.headers._ +import org.http4s.CacheDirective._ +import scala.concurrent.duration._ +import cats._ +import cats.implicits._ +import cats.data._ +import org.typelevel.ci._ + +private[http4s] object CacheRules { + + def requestCanUseCached[F[_]](req: Request[F]): Boolean = + methodIsCacheable(req.method) && + !req.headers.get[`Cache-Control`].exists{ + _.values.exists{ + case `no-cache`(_) => true + case _ => false + } + } + + private val cacheableMethods: Set[Method] = Set( + Method.GET, + Method.HEAD, + // Method.POST // Eventually make this work. + ) + + def methodIsCacheable(m: Method): Boolean = cacheableMethods.contains(m) + + private val cacheableStatus: Set[Status] = Set( + Status.Ok, // 200 + Status.NonAuthoritativeInformation, // 203 + Status.NoContent, // 204 + Status.PartialContent, // 206 + Status.MultipleChoices, // 300 + Status.MovedPermanently, // 301 + // Status.NotModified , // 304 + Status.NotFound, // 404 + Status.MethodNotAllowed, // 405 + Status.Gone, // 410 + Status.UriTooLong, // 414 + Status.NotImplemented, // 501 + ) + + def statusIsCacheable(s: Status): Boolean = cacheableStatus.contains(s) + + def cacheAgeAcceptable[F[_]](req: Request[F], item: CacheItem, now: HttpDate): Boolean = { + req.headers.get[`Cache-Control`] match { + case None => + + // TODO: Investigate how this check works with cache-control + // If the data in the cache is expired and client does not explicitly + // accept stale data, then age is not ok. + item.expires.map(expiresAt => expiresAt >= now).getOrElse(true) + case Some(`Cache-Control`(values)) => + val age = CacheItem.Age.of(item.created, now) + val lifetime = CacheItem.CacheLifetime.of(item.expires, now) + + val maxAgeMet: Boolean = values.toList + .collectFirst{ case c@CacheDirective.`max-age`(_) => c } + .map(maxAge => age.deltaSeconds.seconds <= maxAge.deltaSeconds ) + .getOrElse(true) + + val maxStaleMet: Boolean = { + for { + maxStale <- values.toList.collectFirst{ case c@CacheDirective.`max-stale`(_) => c.deltaSeconds}.flatten + stale <- lifetime + } yield if (stale.deltaSeconds >= 0) true else stale.deltaSeconds.seconds <= maxStale + }.getOrElse(true) + + val minFreshMet: Boolean = { + for { + minFresh <- values.toList.collectFirst{case CacheDirective.`min-fresh`(seconds) => seconds} + expiresAt <- item.expires + } yield (expiresAt.epochSecond - now.epochSecond).seconds <= minFresh + }.getOrElse(true) + + // println(s"Age- $age, Lifetime- $lifetime, maxAgeMet: $maxAgeMet, maxStaleMet: $maxStaleMet, minFreshMet: $minFreshMet") + + maxAgeMet && maxStaleMet && minFreshMet + } + } + + def onlyIfCached[F[_]](req: Request[F]): Boolean = req.headers.get[`Cache-Control`] + .exists{_.values.exists{ + case `only-if-cached` => true + case _ => false + }} + + def cacheControlNoStoreExists[F[_]](response: Response[F]): Boolean = response.headers + .get[`Cache-Control`] + .toList + .flatMap(_.values.toList) + .exists{ + case CacheDirective.`no-store` => true + case _ => false + } + + def cacheControlPrivateExists[F[_]](response: Response[F]): Boolean = response.headers + .get[`Cache-Control`] + .toList + .flatMap(_.values.toList) + .exists{ + case CacheDirective.`private`(_) => true + case _ => false + } + + def authorizationHeaderExists[F[_]](response: Response[F]): Boolean = response.headers + .get[Authorization] + .isDefined + + def cacheControlPublicExists[F[_]](response: Response[F]): Boolean = response.headers + .get[`Cache-Control`] + .toList + .flatMap(_.values.toList) + .exists{ + case CacheDirective.public => true + case _ => false + } + + def mustRevalidate[F[_]](response: Message[F]): Boolean = { + response.headers.get[`Cache-Control`].exists{_.values.exists{ + case CacheDirective.`no-cache`(_) => true + case CacheDirective.`max-age`(age) if age <= 0.seconds => true + case _ => false + }} || response.headers.get(CIString("Pragma")).exists(_.exists(_.value === "no-cache")) + } + + def isCacheable[F[_]](req: Request[F], response: Response[F], cacheType: CacheType): Boolean = { + if (!cacheableMethods.contains(req.method)) { + // println(s"Request Method ${req.method} - not Cacheable") + false + } else if (!statusIsCacheable(response.status)) { + // println(s"Response Status ${response.status} - not Cacheable") + false + } else if (cacheControlNoStoreExists(response)) { + // println("Cache-Control No-Store is present - not Cacheable") + false + } else if (cacheType.isShared && cacheControlPrivateExists(response)) { + // println("Cache is shared and Cache-Control private exists - not Cacheable") + false + } else if (cacheType.isShared && response.headers.get(CIString("Vary")).exists(h => h.exists(_.value === "*"))) { + // println("Cache is shared and Vary header exists as * - not Cacheable") + false + } else if (cacheType.isShared && authorizationHeaderExists(response) && !cacheControlPublicExists(response)) { + // println("Cache is Shared and Authorization Header is present and Cache-Control public is not present - not Cacheable") + false + } else if (mustRevalidate(response) && !(response.headers.get[ETag].isDefined || response.headers.get[`Last-Modified`].isDefined)) { + false + } else if (req.method === Method.GET || req.method === Method.HEAD) { + true + } else if (cacheControlPublicExists(response) || cacheControlPrivateExists(response)) { + true + } else { + response.headers.get[Expires].isDefined + } + } + + def shouldInvalidate[F[_]](request: Request[F], response: Response[F]): Boolean = { + if (Set(Status.NotFound, Status.Gone).contains(response.status)) { + true + } else if (Set(Method.GET, Method.HEAD: Method).contains(request.method)){ + false + } else response.status.isSuccess + } + + def getIfMatch(cachedResponse: CachedResponse): Option[`If-None-Match`] = + cachedResponse.headers.get[ETag].map(_.tag).flatMap{etag => + if (etag.weakness != EntityTag.Weak) `If-None-Match`(NonEmptyList.of(etag).some).some + else None + } + + def getIfUnmodifiedSince(cachedResponse: CachedResponse): Option[`If-Unmodified-Since`] = { + for { + lastModified <- cachedResponse.headers.get[`Last-Modified`] + date <- cachedResponse.headers.get[Date] + _ <- Alternative[Option].guard(date.date.epochSecond - lastModified.date.epochSecond >= 60L) + } yield `If-Unmodified-Since`(lastModified.date) + } + + object FreshnessAndExpiration { + // Age in Seconds + private def getAge[F[_]](now: HttpDate, response: Message[F]): FiniteDuration = { + + // Age Or Zero + val initAgeSeconds: Long = now.epochSecond - + response.headers.get[Date].map(date => date.date.epochSecond) + .getOrElse(0L) + + response.headers.get[Age] + .map(age => Math.max(age.age, initAgeSeconds)) + .getOrElse(initAgeSeconds) + .seconds + } + + // Since We do not emit warnings on cache times over 24 hours, limit cache time + // to max of 24 hours. + private def freshnessLifetime[F[_]](now: HttpDate, response: Message[F]) = { + response.headers.get[`Cache-Control`] + .flatMap{ + case `Cache-Control`(directives) => + directives.collectFirst{ + case `max-age`(deltaSeconds) => + deltaSeconds match { + case finite: FiniteDuration => finite + case _ => 24.hours + } + } + }.orElse{ + for { + exp <- response.headers.get[Expires] + date <- response.headers.get[Date] + } yield (exp.expirationDate.epochSecond - date.date.epochSecond).seconds + }.orElse{ + response.headers.get[`Last-Modified`] + .map{lm => + val estimatedLifetime = (now.epochSecond - lm.date.epochSecond) / 10 + Math.min(24.hours.toSeconds, estimatedLifetime).seconds + } + }.getOrElse(24.hours) + + } + + def getExpires[F[_]](now: HttpDate, response: Message[F]): HttpDate = { + val age = getAge(now, response) + val lifetime = freshnessLifetime(now, response) + val ttl = lifetime - age + + HttpDate.unsafeFromEpochSecond(now.epochSecond + ttl.toSeconds) + } + } + +} \ No newline at end of file diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala new file mode 100644 index 0000000..fdb3ad2 --- /dev/null +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala @@ -0,0 +1,76 @@ +package io.chrisdavenport.mules.http4s.internal + +import io.chrisdavenport.mules.http4s._ +import org.http4s._ +import io.chrisdavenport.mules._ +import cats._ +import cats.syntax.all._ +import cats.data._ +import cats.effect._ +import org.http4s.Header.ToRaw.modelledHeadersToRaw + +private[http4s] class Caching[F[_]: Concurrent: Clock] private[http4s] (cache: Cache[F, (Method, Uri), CacheItem], cacheType: CacheType){ + + def request[G[_]: FlatMap](app: Kleisli[G, Request[F], Response[F]], fk: F ~> G)(req: Request[F]): G[Response[F]] = { + if (CacheRules.requestCanUseCached(req)) { + for { + cachedValue <- fk(cache.lookup((req.method, req.uri))) + now <- fk(HttpDate.current[F]) + out <- cachedValue match { + case None => + if (CacheRules.onlyIfCached(req)) fk(Response[F](Status.GatewayTimeout).pure[F]) + else { + app.run(req) + .flatMap(resp => fk(withResponse(req, resp))) + } + case Some(item) => + if (CacheRules.cacheAgeAcceptable(req, item, now)) { + fk(item.response.toResponse[F].pure[F]) + } else { + app.run( + req + .putHeaders(CacheRules.getIfMatch(item.response).map(modelledHeadersToRaw(_)).toSeq:_*) + .putHeaders(CacheRules.getIfUnmodifiedSince(item.response).map(modelledHeadersToRaw(_)).toSeq:_*) + ).flatMap(resp => fk(withResponse(req, resp))) + } + } + } yield out + } else { + app.run(req) + .flatMap(resp => fk(withResponse(req, resp))) + } + } + + private def withResponse(req: Request[F], resp: Response[F]): F[Response[F]] = { + { + if (CacheRules.shouldInvalidate(req, resp)){ + cache.delete((req.method, req.uri)) + } else Applicative[F].unit + } *> { + if (CacheRules.isCacheable(req, resp, cacheType)){ + for { + cachedResp <- resp.status match { + case Status.NotModified => + cache.lookup((req.method, req.uri)) + .flatMap( + _.map{item => + val cached = item.response + cached.withHeaders(resp.headers ++ cached.headers).pure[F] + } + .getOrElse(CachedResponse.fromResponse[F, F](resp)) + ) + case _ => CachedResponse.fromResponse[F, F](resp) + } + now <- HttpDate.current[F] + expires = CacheRules.FreshnessAndExpiration.getExpires(now, resp) + item <- CacheItem.create(cachedResp, expires.some) + _ <- cache.insert((req.method, req.uri), item) + } yield cachedResp.toResponse[F] + + } else { + resp.pure[F] + } + } + } + +} \ No newline at end of file From a6ea8bab6798ff85c420eb33d305e23e37304613 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Mon, 15 May 2023 19:46:51 +0200 Subject: [PATCH 21/42] add .scalafmt from banana-rdf 3 --- .scalafmt.conf | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/.scalafmt.conf b/.scalafmt.conf index 18962c5..e676fcf 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,7 +1,42 @@ -version = 3.5.9 +version = "3.4.1" runner.dialect = scala3 +indent { + main = 2 + significant = 3 +} +align { + preset = more // For pretty alignment. + stripMargin = true +} +maxColumn = 100 // +assumeStandardLibraryStripMargin = true +rewrite.scala3 { + convertToNewSyntax = true + removeOptionalBraces = yes +} +newlines{ + beforeMultiline = keep + source=keep +} +optIn.breakChainOnFirstMethodDot = false +includeNoParensInSelectChains = false -maxColumn = 100 +optIn.configStyleArguments = true +runner.optimizer.forceConfigStyleMinArgCount = 5 -rewrite.rules = [Imports] -rewrite.imports.sort = scalastyle +fileOverride { + "glob:**.sbt" { + runner.dialect = scala212source3 + } + + "glob:**/project/**.scala" { + runner.dialect = scala212source3 + } + "glob:**/interface/**.scala" { + runner.dialect = scala212source3 + } + + "glob:**/sbt-plugin/**.scala" { + runner.dialect = scala212source3 + } +} From c05858a4291c68889c2e74d8760d0a5914ec81d8 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Tue, 16 May 2023 13:18:00 +0200 Subject: [PATCH 22/42] add CacheType too --- .../mules/http4s/CacheType.scala | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheType.scala diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheType.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheType.scala new file mode 100644 index 0000000..9c69ce5 --- /dev/null +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheType.scala @@ -0,0 +1,23 @@ +package io.chrisdavenport.mules.http4s + +/** + * CacheTypes are in 2 flavors, private caches which are specifically + * for a single user, or public caches which can be used for multiple + * users. Private caches can cache information set to Cache-Control: private, + * whereas public caches are not allowed to cache that information + **/ +sealed trait CacheType { + /** + * Whether or not a Cache is Shared, + * public caches are shared, private caches + * are not + **/ + def isShared: Boolean = this match { + case CacheType.Private => false + case CacheType.Public => true + } +} +object CacheType { + case object Public extends CacheType + case object Private extends CacheType +} \ No newline at end of file From 285c82cae057f1c94934cac80662c8800291c547 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Thu, 18 May 2023 23:39:49 +0200 Subject: [PATCH 23/42] initial wallet with cache tests working --- .../main/scala/net/bblfish/app/Wallet.scala | 28 +- .../main/scala/net/bblfish/wallet/AuthN.scala | 19 +- .../net/bblfish/wallet/BasicAuthWallet.scala | 352 +++++++++--------- .../net/bblfish/web/util/SecurityPrefix.scala | 8 +- .../scala/net/bblfish/wallet/HttpTests.scala | 70 ++-- .../test/scala/net/bblfish/wallet/Jena.scala | 2 +- .../net/bblfish/wallet/WallletToolsTest.scala | 137 ++++--- 7 files changed, 296 insertions(+), 320 deletions(-) diff --git a/wallet/shared/src/main/scala/net/bblfish/app/Wallet.scala b/wallet/shared/src/main/scala/net/bblfish/app/Wallet.scala index 100b0f5..d86fc77 100644 --- a/wallet/shared/src/main/scala/net/bblfish/app/Wallet.scala +++ b/wallet/shared/src/main/scala/net/bblfish/app/Wallet.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,18 +21,18 @@ import org.http4s.{Request, Response} import scala.util.Try -trait Wallet[F[_]] { +trait Wallet[F[_]]: - /** if possible, sign the original request given the information provided by the 40x response. - * - * Note: For this to work, I think we need to assume that the URL in the Request is absolute. see - * [[https://github.com/http4s/http4s/discussions/5930#discussioncomment-3777066 cats-uri discussion]] - */ - def sign(failed: Response[F], lastReq: Request[F]): F[Request[F]] + /** if possible, sign the original request given the information provided by the 40x response. + * + * Note: For this to work, I think we need to assume that the URL in the Request is absolute. + * see + * [[https://github.com/http4s/http4s/discussions/5930#discussioncomment-3777066 cats-uri discussion]] + */ + def sign(failed: Response[F], lastReq: Request[F]): F[Request[F]] - /** previous requests to a server will return acls and methods that can be assumed to be valid - * @param req - * @return - */ - def signFromDB(req: Request[F]): F[Request[F]] -} + /** previous requests to a server will return acls and methods that can be assumed to be valid + * @param req + * @return + */ + def signFromDB(req: Request[F]): F[Request[F]] diff --git a/wallet/shared/src/main/scala/net/bblfish/wallet/AuthN.scala b/wallet/shared/src/main/scala/net/bblfish/wallet/AuthN.scala index 6c5a8b1..f8f762e 100644 --- a/wallet/shared/src/main/scala/net/bblfish/wallet/AuthN.scala +++ b/wallet/shared/src/main/scala/net/bblfish/wallet/AuthN.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,14 +28,13 @@ import org.http4s.headers.Authorization * ASync monad for time it takes to get signature - could be Id monad on many platforms */ trait AuthN[F[_], S[_]]: - /** Signing the request This could be a function, but on some platforms and for some signing - * algorithms the signing may need to be asynchronous in S[_]. The internal monad F on the other - * had will be asyncrhonous, as it is needed for streaming responses. - */ - def sign(originalReq: Request[F]): S[Request[F]] + /** Signing the request This could be a function, but on some platforms and for some signing + * algorithms the signing may need to be asynchronous in S[_]. The internal monad F on the other + * had will be asyncrhonous, as it is needed for streaming responses. + */ + def sign(originalReq: Request[F]): S[Request[F]] class Basic[F[_]](username: String, pass: String) extends AuthN[F, Id]: - override def sign(originalReq: Request[F]): Id[Request[F]] = - originalReq.withHeaders( - Authorization(org.http4s.BasicCredentials(username, pass)) - ) + override def sign(originalReq: Request[F]): Id[Request[F]] = originalReq.withHeaders( + Authorization(org.http4s.BasicCredentials(username, pass)) + ) diff --git a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala index 8375df5..e759644 100644 --- a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala +++ b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -65,90 +65,82 @@ class KeyData[F[_]]( // import ops.{given, *} // lazy val keyIdAtt = KeyId(Rfc8941.SfString(keyId.value)) - def mkSigInput(now: FiniteDuration): ReqSigInput[H4] = - import run.cosy.http.headers.SigIn.* - ReqSigInput[H4]()(Created(now.toSeconds), keyIdAtt) + def mkSigInput(now: FiniteDuration): ReqSigInput[H4] = + import run.cosy.http.headers.SigIn.* + ReqSigInput[H4]()(Created(now.toSeconds), keyIdAtt) end KeyData trait ChallengeResponse: - // the challenge scheme for which this response is designed - def forChallengeScheme: String + // the challenge scheme for which this response is designed + def forChallengeScheme: String - // - def respondTo[F[_]]( - remote: ll.AbsoluteUrl, // request for original Host - originalRequest: h4s.Request[F], - response: h4s.Response[F] - ): F[h4s.Request[F]] + // + def respondTo[F[_]]( + remote: ll.AbsoluteUrl, // request for original Host + originalRequest: h4s.Request[F], + response: h4s.Response[F] + ): F[h4s.Request[F]] object BasicWallet: - val effectiveAclLink = "effectiveAccessControl" - val EffectiveAclOpt = Some(effectiveAclLink) - val aclRelTypes = List(EffectiveAclOpt, "acl", effectiveAclLink) + val effectiveAclLink = "effectiveAccessControl" + val EffectiveAclOpt = Some(effectiveAclLink) + val aclRelTypes = List(EffectiveAclOpt, "acl", effectiveAclLink) /** place code that only needs RDF and ops here. */ class WalletTools[Rdf <: RDF](using ops: Ops[Rdf]): - import run.cosy.web.util.UrlUtil.* - import ops.{*, given} + import run.cosy.web.util.UrlUtil.* + import ops.{*, given} - val wac: WebACL[Rdf] = WebACL[Rdf] - val foaf = prefix.FOAF[Rdf] - val sec: SecurityPrefix[Rdf] = SecurityPrefix[Rdf] + val wac: WebACL[Rdf] = WebACL[Rdf] + val foaf = prefix.FOAF[Rdf] + val sec: SecurityPrefix[Rdf] = SecurityPrefix[Rdf] - def withinTry(requestUri: RDF.URI[Rdf], container: RDF.URI[Rdf]): Try[Boolean] = - for - case requ: ll.AbsoluteUrl <- requestUri.toLL - case ctnrU: ll.AbsoluteUrl <- container.toLL - yield - val se = requ.schemeOption == ctnrU.schemeOption - val ae = requ.authorityOption == ctnrU.authorityOption - val rp = requ.path.parts - // because ll.Url interprets the https://foo.bar/ as having a path with one empty string - val cp = ctnrU.path.parts - val cpClean = if cp.last == "" then cp.dropRight(1) else cp - val pe = rp.startsWith(cpClean) - se && ae && pe + def withinTry(requestUri: RDF.URI[Rdf], container: RDF.URI[Rdf]): Try[Boolean] = + for + case requ: ll.AbsoluteUrl <- requestUri.toLL + case ctnrU: ll.AbsoluteUrl <- container.toLL + yield + val se = requ.schemeOption == ctnrU.schemeOption + val ae = requ.authorityOption == ctnrU.authorityOption + val rp = requ.path.parts + // because ll.Url interprets the https://foo.bar/ as having a path with one empty string + val cp = ctnrU.path.parts + val cpClean = if cp.last == "" then cp.dropRight(1) else cp + val pe = rp.startsWith(cpClean) + se && ae && pe - extension (uri: RDF.URI[Rdf]) - def contains(longer: RDF.URI[Rdf]): Boolean = - withinTry(longer, uri).getOrElse(false) + extension (uri: RDF.URI[Rdf]) + def contains(longer: RDF.URI[Rdf]): Boolean = withinTry(longer, uri).getOrElse(false) - def findAclsFor(g: RDF.Graph[Rdf], requestUri: RDF.URI[Rdf]): Iterator[St.Subject[Rdf]] = - import run.cosy.web.util.UrlUtil.* - val directRules: Iterator[St.Subject[Rdf]] = g.find(`*`, wac.accessTo, requestUri).map(_.subj) - val defaultRules: Iterator[St.Subject[Rdf]] = g.find(`*`, wac.default, `*`).collect { - case ops.Triple(rule, _, defaultContainer: RDF.URI[Rdf]) - if defaultContainer.contains(requestUri) => - rule - } - directRules ++ defaultRules + def findAclsFor(g: RDF.Graph[Rdf], requestUri: RDF.URI[Rdf]): Iterator[St.Subject[Rdf]] = + import run.cosy.web.util.UrlUtil.* + val directRules: Iterator[St.Subject[Rdf]] = g.find(`*`, wac.accessTo, requestUri).map(_.subj) + val defaultRules: Iterator[St.Subject[Rdf]] = g.find(`*`, wac.default, `*`).collect { + case ops.Triple(rule, _, defaultContainer: RDF.URI[Rdf]) + if defaultContainer.contains(requestUri) => rule + } + directRules ++ defaultRules - def modeForMethod(method: h4s.Method): RDF.URI[Rdf] = - import h4s.Method.{GET, HEAD, SEARCH} - if List(GET, HEAD, SEARCH).contains(method) then wac.Read - else wac.Write + def modeForMethod(method: h4s.Method): RDF.URI[Rdf] = + import h4s.Method.{GET, HEAD, SEARCH} + if List(GET, HEAD, SEARCH).contains(method) then wac.Read + else wac.Write - def findAgents( - aclGr: RDF.Graph[Rdf], - reqUrl: RDF.URI[Rdf], - mode: h4s.Method - ): Iterator[St.Object[Rdf]] = { - for - ruleNode <- findAclsFor(aclGr, reqUrl) - if !aclGr - .find(ruleNode, wac.mode, modeForMethod(mode)) - .isEmpty - obj <- aclGr - .find(ruleNode, wac.agent, `*`) - .map(_.obj) - .collect[St.Subject[Rdf]] { - case u: RDF.URI[Rdf] => u + def findAgents( + aclGr: RDF.Graph[Rdf], + reqUrl: RDF.URI[Rdf], + mode: h4s.Method + ): Iterator[St.Object[Rdf]] = + for + ruleNode <- findAclsFor(aclGr, reqUrl) + if !aclGr.find(ruleNode, wac.mode, modeForMethod(mode)).isEmpty + obj <- aclGr.find(ruleNode, wac.agent, `*`).map(_.obj).collect[St.Subject[Rdf]] { + case u: RDF.URI[Rdf] => u case b: RDF.BNode[Rdf] => b // todo: the key could be in a literal too !? } - yield obj - } + yield obj end WalletTools /** First attempt at a Wallet, just to get things going. The wallet must be given collections of @@ -179,149 +171,137 @@ class BasicWallet[F[_], Rdf <: RDF]( fc: Concurrent[F], clock: Clock[F] ) extends Wallet[F]: - val wt: WalletTools[Rdf] = new WalletTools[Rdf] + val wt: WalletTools[Rdf] = new WalletTools[Rdf] - val reqSel: ReqSelectors[H4] = new ReqSelectors[H4](using new SelectorFnsH4()) - import ops.{*, given} - import rdfDecoders.allrdf - import reqSel.* - import reqSel.RequestHd.* - import run.cosy.http4s.Http4sTp - import run.cosy.http4s.Http4sTp.{*, given} + val reqSel: ReqSelectors[H4] = new ReqSelectors[H4](using new SelectorFnsH4()) + import ops.{*, given} + import rdfDecoders.allrdf + import reqSel.* + import reqSel.RequestHd.* + import run.cosy.http4s.Http4sTp + import run.cosy.http4s.Http4sTp.{*, given} - import scala.language.implicitConversions - import wt.* + import scala.language.implicitConversions + import wt.* - def reqToH4Req(h4req: h4s.Request[F]): Http.Request[H4] = - h4req.asInstanceOf[Http.Request[H4]] + def reqToH4Req(h4req: h4s.Request[F]): Http.Request[H4] = h4req.asInstanceOf[Http.Request[H4]] - def h4ReqToHttpReq(h4req: Http.Request[H4]): h4s.Request[F] = - h4req.asInstanceOf[h4s.Request[F]] + def h4ReqToHttpReq(h4req: Http.Request[H4]): h4s.Request[F] = h4req.asInstanceOf[h4s.Request[F]] - // todo: here we assume the request Uri usually has the Host. Need to verify, or pass the full Url - def basicChallenge( - host: ll.Authority, // request for original Host - originalRequest: h4s.Request[F], - nel: NonEmptyList[h4s.Challenge] - ): Try[h4s.Request[F]] = - // todo: can one use some of the info in the Challenge? - val either = for - _ <- nel - .find(_.scheme == "Basic") - .toRight(Exception("server does not make basic auth available")) - id <- db.get(host).toRight(Exception("no passwords for hot " + host)) - yield originalRequest.withHeaders( - Authorization(h4s.BasicCredentials(id.username, id.password)) - ) - either.toTry - end basicChallenge + // todo: here we assume the request Uri usually has the Host. Need to verify, or pass the full Url + def basicChallenge( + host: ll.Authority, // request for original Host + originalRequest: h4s.Request[F], + nel: NonEmptyList[h4s.Challenge] + ): Try[h4s.Request[F]] = + // todo: can one use some of the info in the Challenge? + val either = + for + _ <- nel.find(_.scheme == "Basic") + .toRight(Exception("server does not make basic auth available")) + id <- db.get(host).toRight(Exception("no passwords for hot " + host)) + yield originalRequest.withHeaders( + Authorization(h4s.BasicCredentials(id.username, id.password)) + ) + either.toTry + end basicChallenge - def httpSigChallenge( - requestUrl: ll.AbsoluteUrl, // http4s.Request objects are not guaranteed to contain absolute urls - originalRequest: h4s.Request[F], - response: h4s.Response[F], - nel: NonEmptyList[h4s.Challenge] - ): F[h4s.Request[F]] = - import BasicWallet.* - val aclLinks: List[LinkValue] = - for - httpSig <- nel.find(_.scheme == "HttpSig").toList - link <- response.headers.get[Link].toList - linkVal <- link.values.toList - rel <- linkVal.rel.toList - if aclRelTypes.contains(rel) - yield linkVal - aclLinks - .find(_.rel == EffectiveAclOpt) - .orElse(aclLinks.headOption) match - case None => - fc.raiseError(Exception("no acl Link in header. Cannot find where the rules are.")) - case Some(linkVal) => - val h4req = llUrltoHttp4s(requestUrl) - val absLink: h4s.Uri = h4req.resolve(linkVal.uri) - client - .fetchAs[RDF.rGraph[Rdf]]( - h4s.Request( - uri = absLink.withoutFragment - ) - ) - .flatMap { (rG: RDF.rGraph[Rdf]) => + def httpSigChallenge( + requestUrl: ll.AbsoluteUrl, // http4s.Request objects are not guaranteed to contain absolute urls + originalRequest: h4s.Request[F], + response: h4s.Response[F], + nel: NonEmptyList[h4s.Challenge] + ): F[h4s.Request[F]] = + import BasicWallet.* + val aclLinks: List[LinkValue] = + for + httpSig <- nel.find(_.scheme == "HttpSig").toList + link <- response.headers.get[Link].toList + linkVal <- link.values.toList + rel <- linkVal.rel.toList + if aclRelTypes.contains(rel) + yield linkVal + aclLinks.find(_.rel == EffectiveAclOpt).orElse(aclLinks.headOption) match + case None => fc + .raiseError(Exception("no acl Link in header. Cannot find where the rules are.")) + case Some(linkVal) => + val h4req = llUrltoHttp4s(requestUrl) + val absLink: h4s.Uri = h4req.resolve(linkVal.uri) + client.fetchAs[RDF.rGraph[Rdf]]( + h4s.Request( + uri = absLink.withoutFragment + ) + ).flatMap { (rG: RDF.rGraph[Rdf]) => // todo: what if original url is relative? val lllink = http4sUrlToLLUrl(absLink).toAbsoluteUrl val g: RDF.Graph[Rdf] = rG.resolveAgainst(lllink) val reqRes = ops.URI(originalRequest.uri.toString) // <-- // this requires all the info to be in the same graph. Needs generalisation to // jump across graphs - val keyNodes: Iterator[St.Subject[Rdf]] = for - agentNode <- findAgents(g, reqRes, originalRequest.method) - controllerTriple <- g.find(*, sec.controller, agentNode) - yield controllerTriple.subj + val keyNodes: Iterator[St.Subject[Rdf]] = + for + agentNode <- findAgents(g, reqRes, originalRequest.method) + controllerTriple <- g.find(*, sec.controller, agentNode) + yield controllerTriple.subj import run.cosy.http4s.Http4sTp.given - val keys: Iterable[KeyData[F]] = keyNodes - .collect { case u: RDF.URI[Rdf] => - keyIdDB.find(kid => kid.keyIdAtt.value.asciiStr == u.value).toList - } - .flatten - .to(Iterable) + val keys: Iterable[KeyData[F]] = keyNodes.collect { case u: RDF.URI[Rdf] => + keyIdDB.find(kid => kid.keyIdAtt.value.asciiStr == u.value).toList + }.flatten.to(Iterable) for - keydt <- fc.fromOption[KeyData[F]]( - keys.headOption, - Exception( - s"none of our keys fit the ACL $lllink for resource $reqRes accessed in " + - s"${originalRequest.method} matches the rules in { $g } " - ) - ) - signingFn <- keydt.signer - now <- clock.realTime // <- todo, add clock time caching perhaps - signedReq <- MessageSignature.withSigInput[F, H4]( - originalRequest.asInstanceOf[Http.Request[H4]], - Rfc8941.Token("sig1"), - keydt.mkSigInput(now), - signingFn - ) + keydt <- fc.fromOption[KeyData[F]]( + keys.headOption, + Exception( + s"none of our keys fit the ACL $lllink for resource $reqRes accessed in " + + s"${originalRequest.method} matches the rules in { $g } " + ) + ) + signingFn <- keydt.signer + now <- clock.realTime // <- todo, add clock time caching perhaps + signedReq <- MessageSignature.withSigInput[F, H4]( + originalRequest.asInstanceOf[Http.Request[H4]], + Rfc8941.Token("sig1"), + keydt.mkSigInput(now), + signingFn + ) yield - val res = run.cosy.http4s.Http4sTp.hOps - .addHeader[Http.Request[H4]](signedReq)("Authorization", "HttpSig proof=sig1") - h4ReqToHttpReq(res) - } - end httpSigChallenge + val res = run.cosy.http4s.Http4sTp.hOps + .addHeader[Http.Request[H4]](signedReq)("Authorization", "HttpSig proof=sig1") + h4ReqToHttpReq(res) + } + end httpSigChallenge - /** This is different from middleware such as FollowRedirects, as that essentially continues the - * request. Here we need to stop the request and make new ones to find the access control rules - * for the given resource. (that could just be a BasicAuth request for a password, or a more - * complex description linked to from the resource, but it may also just involve querying a DB or - * quad store.) - */ - override def sign( - failed: h4s.Response[F], - lastReq: h4s.Request[F] - ): F[h4s.Request[F]] = - import cats.syntax.applicativeError.given - failed.status.code match - case 402 => - fc.raiseError( - new Exception("We don't support payment authentication yet") - ) - case 401 => - failed.headers.get[h4hdr.`WWW-Authenticate`] match - case None => - fc.raiseError( + /** This is different from middleware such as FollowRedirects, as that essentially continues the + * request. Here we need to stop the request and make new ones to find the access control rules + * for the given resource. (that could just be a BasicAuth request for a password, or a more + * complex description linked to from the resource, but it may also just involve querying a DB + * or quad store.) + */ + override def sign( + failed: h4s.Response[F], + lastReq: h4s.Request[F] + ): F[h4s.Request[F]] = + import cats.syntax.applicativeError.given + failed.status.code match + case 402 => fc.raiseError( + new Exception("We don't support payment authentication yet") + ) + case 401 => failed.headers.get[h4hdr.`WWW-Authenticate`] match + case None => fc.raiseError( new Exception("No WWW-Authenticate header. Don't know how to login") ) case Some(h4hdr.`WWW-Authenticate`(nel)) => // do we recognise a method? for - url <- fc.fromTry(Try(http4sUrlToLLUrl(lastReq.uri).toAbsoluteUrl)) - authdReq <- fc - .fromTry(basicChallenge(url.authority, lastReq, nel)) - .handleErrorWith { _ => - httpSigChallenge(url, lastReq, failed, nel) - } + url <- fc.fromTry(Try(http4sUrlToLLUrl(lastReq.uri).toAbsoluteUrl)) + authdReq <- fc.fromTry(basicChallenge(url.authority, lastReq, nel)) + .handleErrorWith { _ => + httpSigChallenge(url, lastReq, failed, nel) + } yield authdReq - case _ => ??? // fail - end sign + case _ => ??? // fail + end sign - override def signFromDB(req: h4s.Request[F]): F[h4s.Request[F]] = fc.point(req) + override def signFromDB(req: h4s.Request[F]): F[h4s.Request[F]] = fc.point(req) end BasicWallet diff --git a/wallet/shared/src/main/scala/net/bblfish/web/util/SecurityPrefix.scala b/wallet/shared/src/main/scala/net/bblfish/web/util/SecurityPrefix.scala index 4438067..e7f1cb1 100644 --- a/wallet/shared/src/main/scala/net/bblfish/web/util/SecurityPrefix.scala +++ b/wallet/shared/src/main/scala/net/bblfish/web/util/SecurityPrefix.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,7 @@ package net.bblfish.web.util import org.w3.banana.{Ops, PrefixBuilder, RDF} object SecurityPrefix: - def apply[Rdf <: RDF](using ops: Ops[Rdf]) = new SecurityPrefix[Rdf] + def apply[Rdf <: RDF](using ops: Ops[Rdf]) = new SecurityPrefix[Rdf] /** Note: the security prefix https://w3id.org/security/v1# is not a namespace! That is a context * document for rdfa, containing shortcuts for many different names coming from different @@ -35,7 +35,7 @@ class SecurityPrefix[Rdf <: RDF](using val ops: Ops[Rdf]) ops.URI("https://w3id.org/security#") ): - val controller = apply("controller") - val publicKeyJwk = apply("publicKeyJwk") + val controller = apply("controller") + val publicKeyJwk = apply("publicKeyJwk") end SecurityPrefix diff --git a/wallet/shared/src/test/scala/net/bblfish/wallet/HttpTests.scala b/wallet/shared/src/test/scala/net/bblfish/wallet/HttpTests.scala index cc96bc8..fd08b74 100644 --- a/wallet/shared/src/test/scala/net/bblfish/wallet/HttpTests.scala +++ b/wallet/shared/src/test/scala/net/bblfish/wallet/HttpTests.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,38 +19,36 @@ package net.bblfish.wallet import org.http4s.{ParseResult, Uri} import org.http4s.headers.{Link, LinkValue} -class HttpTests extends munit.FunSuite { - - test( - ("parse Link header. " + - "ignore because of https://github.com/http4s/http4s/issues/7101").ignore - ) { - val ht1 = """<>; rel=https://www.w3.org/ns/auth/acl#accessControl""" - val ht1res: ParseResult[Link] = Link.parse(ht1) - assert(ht1res.isLeft, ht1res) - } - - test("parse Link header working") { - val h1 = """; rel=acl,""" + - """ ; rel="https://www.w3.org/ns/auth/acl#accessControl"""" - val pr1: ParseResult[Link] = Link.parse(h1) - assert(pr1.isRight, pr1) - - val h2 = """; rel=acl,""" + - """ ; rel=https://www.w3.org/ns/auth/acl#accessControl""" - val pr2: ParseResult[Link] = Link.parse(h2) - assert(pr2.isLeft, pr2) - - val rfc8288Ex = """; rel="start http://example.net/relation/other"""" - val Right(Link(values)) = Link.parse(rfc8288Ex): @unchecked - assertEquals(values.size, 1) - assertEquals( - values.head, - LinkValue( - Uri.unsafeFromString("http://example.org/"), - rel = Some("start http://example.net/relation/other") - ) - ) - } - -} +class HttpTests extends munit.FunSuite: + + test( + ("parse Link header. " + + "ignore because of https://github.com/http4s/http4s/issues/7101").ignore + ) { + val ht1 = """<>; rel=https://www.w3.org/ns/auth/acl#accessControl""" + val ht1res: ParseResult[Link] = Link.parse(ht1) + assert(ht1res.isLeft, ht1res) + } + + test("parse Link header working") { + val h1 = """; rel=acl,""" + + """ ; rel="https://www.w3.org/ns/auth/acl#accessControl"""" + val pr1: ParseResult[Link] = Link.parse(h1) + assert(pr1.isRight, pr1) + + val h2 = """; rel=acl,""" + + """ ; rel=https://www.w3.org/ns/auth/acl#accessControl""" + val pr2: ParseResult[Link] = Link.parse(h2) + assert(pr2.isLeft, pr2) + + val rfc8288Ex = """; rel="start http://example.net/relation/other"""" + val Right(Link(values)) = Link.parse(rfc8288Ex): @unchecked + assertEquals(values.size, 1) + assertEquals( + values.head, + LinkValue( + Uri.unsafeFromString("http://example.org/"), + rel = Some("start http://example.net/relation/other") + ) + ) + } diff --git a/wallet/shared/src/test/scala/net/bblfish/wallet/Jena.scala b/wallet/shared/src/test/scala/net/bblfish/wallet/Jena.scala index 005379c..cf3d2a9 100644 --- a/wallet/shared/src/test/scala/net/bblfish/wallet/Jena.scala +++ b/wallet/shared/src/test/scala/net/bblfish/wallet/Jena.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/wallet/shared/src/test/scala/net/bblfish/wallet/WallletToolsTest.scala b/wallet/shared/src/test/scala/net/bblfish/wallet/WallletToolsTest.scala index 27192c6..bb7e359 100644 --- a/wallet/shared/src/test/scala/net/bblfish/wallet/WallletToolsTest.scala +++ b/wallet/shared/src/test/scala/net/bblfish/wallet/WallletToolsTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,71 +24,70 @@ import io.lemonlabs.uri as ll import org.http4s.Method.GET import org.w3.banana.RDF.Statement as St -trait WalletToolsTest[R <: RDF](using ops: Ops[R]) extends munit.FunSuite { - val wt = new WalletTools[R] - import ops.{*, given} - import wt.* - import org.w3.banana.diesel.{*, given} - - val acl1: RDF.rGraph[R] = (rURI("#R0") -- rdf.typ ->- wac.Authorization - -- wac.mode ->- wac.Control - -- wac.agent ->- BNode("a")).graph ++ ( - rURI("#R1") -- rdf.`type` ->- wac.Authorization - -- wac.mode ->- wac.Read - -- wac.default ->- rURI(".") - -- wac.agent ->- rURI("#a") // we use a URI here for testAgents - ).graph.triples.toSeq ++ ( - rURI("#R2") -- rdf.`type` ->- wac.Authorization - -- wac.mode ->- wac.Write - -- wac.default ->- rURI(".") - -- wac.agent ->- rURI("#a") - ).graph.triples.toSeq ++ ( - rURI("/rfcKey#") -- sec.controller ->- wac.Write - ).graph.triples.toSeq - - val bbl = URI("https://bblfish.net/") - val bblPpl = URI("https://bblfish.net/people") - val bblPplS = URI("https://bblfish.net/people/") - val bblCard = URI("https://bblfish.net/people/henry/card") - val alice = URI("https://alice.name/card#me") - - test("WalletTools.within") { - assertEquals(wt.withinTry(bblCard, bbl), Success(true)) - assertEquals(wt.withinTry(bbl, bblCard), Success(false)) - assertEquals(wt.withinTry(bblCard, bblPpl), Success(true)) - assertEquals(wt.withinTry(bblCard, bblPplS), Success(true)) - - assertEquals(bbl.contains(bblPpl), true) - assertEquals(bbl.contains(bblPplS), true) - assertEquals(bblPplS.contains(bblPplS), true) - assertEquals(bblPplS.contains(bblCard), true) - } - - val bblRootAcl = ll.AbsoluteUrl.parse("https://bblfish.net/.acl") - val acl1Gr: RDF.Graph[R] = acl1.resolveAgainst(bblRootAcl) - - test("test find acl") { - val r1r2: Set[St.Subject[R]] = - Set(URI("https://bblfish.net/.acl#R1"), URI("https://bblfish.net/.acl#R2")) - - val n1: Set[St.Subject[R]] = findAclsFor(acl1Gr, bbl).toSet - assertEquals(n1, r1r2) - - val n2: Set[St.Subject[R]] = findAclsFor(acl1Gr, bblPplS).toSet - assertEquals(n2, r1r2) - - val n3: Set[St.Subject[R]] = findAclsFor(acl1Gr, bblCard).toSet - assertEquals(n3, r1r2) - - val n4: List[St.Subject[R]] = findAclsFor(acl1Gr, alice).toList - assertEquals(n4, List()) - - // todo: add code for the wac: - } - - test("findAgents") { - assertEquals(findAgents(acl1Gr, bblCard, GET).toList, List(URI("https://bblfish.net/.acl#a"))) - - } - -} +trait WalletToolsTest[R <: RDF](using ops: Ops[R]) extends munit.FunSuite: + val wt = new WalletTools[R] + import ops.{*, given} + import wt.* + import org.w3.banana.diesel.{*, given} + + val acl1: RDF.rGraph[R] = + (rURI("#R0") -- rdf.typ ->- wac.Authorization + -- wac.mode ->- wac.Control + -- wac.agent ->- BNode("a")).graph ++ ( + rURI("#R1") -- rdf.`type` ->- wac.Authorization + -- wac.mode ->- wac.Read + -- wac.default ->- rURI(".") + -- wac.agent ->- rURI("#a") // we use a URI here for testAgents + ).graph.triples.toSeq ++ ( + rURI("#R2") -- rdf.`type` ->- wac.Authorization + -- wac.mode ->- wac.Write + -- wac.default ->- rURI(".") + -- wac.agent ->- rURI("#a") + ).graph.triples.toSeq ++ ( + rURI("/rfcKey#") -- sec.controller ->- wac.Write + ).graph.triples.toSeq + + val bbl = URI("https://bblfish.net/") + val bblPpl = URI("https://bblfish.net/people") + val bblPplS = URI("https://bblfish.net/people/") + val bblCard = URI("https://bblfish.net/people/henry/card") + val alice = URI("https://alice.name/card#me") + + test("WalletTools.within") { + assertEquals(wt.withinTry(bblCard, bbl), Success(true)) + assertEquals(wt.withinTry(bbl, bblCard), Success(false)) + assertEquals(wt.withinTry(bblCard, bblPpl), Success(true)) + assertEquals(wt.withinTry(bblCard, bblPplS), Success(true)) + + assertEquals(bbl.contains(bblPpl), true) + assertEquals(bbl.contains(bblPplS), true) + assertEquals(bblPplS.contains(bblPplS), true) + assertEquals(bblPplS.contains(bblCard), true) + } + + val bblRootAcl = ll.AbsoluteUrl.parse("https://bblfish.net/.acl") + val acl1Gr: RDF.Graph[R] = acl1.resolveAgainst(bblRootAcl) + + test("test find acl") { + val r1r2: Set[St.Subject[R]] = + Set(URI("https://bblfish.net/.acl#R1"), URI("https://bblfish.net/.acl#R2")) + + val n1: Set[St.Subject[R]] = findAclsFor(acl1Gr, bbl).toSet + assertEquals(n1, r1r2) + + val n2: Set[St.Subject[R]] = findAclsFor(acl1Gr, bblPplS).toSet + assertEquals(n2, r1r2) + + val n3: Set[St.Subject[R]] = findAclsFor(acl1Gr, bblCard).toSet + assertEquals(n3, r1r2) + + val n4: List[St.Subject[R]] = findAclsFor(acl1Gr, alice).toList + assertEquals(n4, List()) + + // todo: add code for the wac: + } + + test("findAgents") { + assertEquals(findAgents(acl1Gr, bblCard, GET).toList, List(URI("https://bblfish.net/.acl#a"))) + + } From feeaca60612688c99c035766dfc4857357fe5001 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Thu, 18 May 2023 23:41:04 +0200 Subject: [PATCH 24/42] scalafmt added and applied everywhere --- .scalafmt.conf | 23 +- .../net/bblfish/app/auth/AuthNClient.scala | 47 +- .../cosy/solid/app/auth/AuthNClientTest.scala | 311 +++++++------ build.sbt | 63 +-- .../mules/http4s/CacheItem.scala | 67 +-- .../mules/http4s/CacheType.scala | 46 +- .../mules/http4s/CachedResponse.scala | 91 ++-- .../mules/http4s/internal/CacheRules.scala | 419 +++++++++--------- .../mules/http4s/internal/Caching.scala | 153 ++++--- .../scala/run/cosy/http/cache/Cache.scala | 134 +++--- .../scala/run/cosy/http/cache/DirTree.scala | 210 ++++----- .../cache/InterpretedCacheMiddleware.scala | 50 +++ .../scala/run/cosy/http/cache/CacheTest.scala | 150 ++++--- .../run/cosy/http/cache/DirTreeTest.scala | 194 ++++---- .../cache/InterpretedCacheMiddleTest.scala | 117 +++++ .../test/scala/run/cosy/http/cache/Web.scala | 149 +++++++ .../run/cosy/ld/http4s/RDFDecoders.scala | 96 ++-- .../scala/run/cosy/web/util/UrlUtil.scala | 83 ++-- .../src/main/scala/run/cosy/ld/PNGraph.scala | 224 +++++----- .../main/scala/run/cosy/ld/http4s/H4Web.scala | 34 +- .../main/scala/run/cosy/ldes/LdesSpider.scala | 110 +++-- .../scala/run/cosy/ldes/prefix/LDES.scala | 28 +- .../scala/run/cosy/ldes/prefix/SOSA.scala | 76 ++-- .../scala/run/cosy/ldes/prefix/TREE.scala | 64 +-- .../scala/run/cosy/ldes/prefix/WGS84.scala | 16 +- .../test/scala/run/cosy/ld/FoafWebTest.scala | 147 +++--- .../test/scala/run/cosy/ld/JenaWebTest.scala | 2 +- .../scala/run/cosy/ld/LdesBrokenWebTest.scala | 142 +++--- .../scala/run/cosy/ld/LdesSimpleWebTest.scala | 209 ++++----- .../test/scala/run/cosy/ld/MiniFoafWWW.scala | 87 ++-- .../run/cosy/ld/ldes/BrokenMiniLdesWWW.scala | 85 ++-- .../scala/run/cosy/ld/ldes/MiniLdesWWW.scala | 354 ++++++++------- project/Dependencies.scala | 32 +- project/build.properties | 2 +- .../main/scala/scripts/AnHttpSigClient.scala | 65 ++- .../src/main/scala/scripts/MiniCF.scala | 85 ++-- 36 files changed, 2229 insertions(+), 1936 deletions(-) create mode 100644 cache/shared/src/main/scala/run/cosy/http/cache/InterpretedCacheMiddleware.scala create mode 100644 cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala create mode 100644 cache/shared/src/test/scala/run/cosy/http/cache/Web.scala diff --git a/.scalafmt.conf b/.scalafmt.conf index e676fcf..b7254eb 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,29 +1,24 @@ -version = "3.4.1" +version = "3.7.3" runner.dialect = scala3 indent { main = 2 + matchSite = 1 significant = 3 } align { - preset = more // For pretty alignment. - stripMargin = true + preset = none + stripMargin = false } -maxColumn = 100 // +maxColumn = 100 assumeStandardLibraryStripMargin = true rewrite.scala3 { convertToNewSyntax = true removeOptionalBraces = yes } -newlines{ - beforeMultiline = keep - source=keep +newlines { + selectChains = fold + beforeMultiline = fold } -optIn.breakChainOnFirstMethodDot = false -includeNoParensInSelectChains = false - -optIn.configStyleArguments = true -runner.optimizer.forceConfigStyleMinArgCount = 5 - fileOverride { "glob:**.sbt" { runner.dialect = scala212source3 @@ -39,4 +34,4 @@ fileOverride { "glob:**/sbt-plugin/**.scala" { runner.dialect = scala212source3 } -} +} \ No newline at end of file diff --git a/authn/shared/src/main/scala/net/bblfish/app/auth/AuthNClient.scala b/authn/shared/src/main/scala/net/bblfish/app/auth/AuthNClient.scala index ed68d81..7c3fe3d 100644 --- a/authn/shared/src/main/scala/net/bblfish/app/auth/AuthNClient.scala +++ b/authn/shared/src/main/scala/net/bblfish/app/auth/AuthNClient.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,36 +37,35 @@ import scala.util.{Failure, Success, Try} * Wallet to have requests signed. */ object AuthNClient: - def apply[F[_]: Concurrent](wallet: Wallet[F])( - client: Client[F] - ): Client[F] = + def apply[F[_]: Concurrent](wallet: Wallet[F])( + client: Client[F] + ): Client[F] = - def authLoop( - req: Request[F], - attempts: Int, - hotswap: Hotswap[F, Response[F]] - ): F[Response[F]] = - hotswap.clear *> // Release the prior connection before allocating a new + def authLoop( + req: Request[F], + attempts: Int, + hotswap: Hotswap[F, Response[F]] + ): F[Response[F]] = hotswap.clear *> // Release the prior connection before allocating a new // todo: we should enhance the req with a signature if we already have info on the server hotswap.swap(client.run(req)).flatMap { (resp: Response[F]) => // todo: may want a lot more flexibility than attempt numbering to determine if we should retry or not. resp.status match - case Status.Unauthorized if attempts < 1 => - wallet.sign(resp, req).flatMap(newReq => authLoop(newReq, attempts + 1, hotswap)) - case _ => resp.pure[F] + case Status.Unauthorized if attempts < 1 => + wallet.sign(resp, req).flatMap(newReq => authLoop(newReq, attempts + 1, hotswap)) + case _ => resp.pure[F] } - Client { req => - // using the pattern from FollowRedirect example using Hotswap. - // Not 100% sure this is so much needed here... - Hotswap.create[F, Response[F]].flatMap { hotswap => - Resource.eval( - wallet.signFromDB(req).flatMap { possiblySignedReq => - authLoop(possiblySignedReq, 0, hotswap) - } - ) + Client { req => + // using the pattern from FollowRedirect example using Hotswap. + // Not 100% sure this is so much needed here... + Hotswap.create[F, Response[F]].flatMap { hotswap => + Resource.eval( + wallet.signFromDB(req).flatMap { possiblySignedReq => + authLoop(possiblySignedReq, 0, hotswap) + } + ) + } } - } - end apply + end apply end AuthNClient diff --git a/authn/shared/src/test/scala/run/cosy/solid/app/auth/AuthNClientTest.scala b/authn/shared/src/test/scala/run/cosy/solid/app/auth/AuthNClientTest.scala index 346b5ab..cfff23f 100644 --- a/authn/shared/src/test/scala/run/cosy/solid/app/auth/AuthNClientTest.scala +++ b/authn/shared/src/test/scala/run/cosy/solid/app/auth/AuthNClientTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -49,161 +49,154 @@ import run.cosy.ld.http4s.RDFDecoders case class User(id: Long, name: String) -class AuthNClientTest extends munit.CatsEffectSuite { - - val realm = "Test Realm" - val username = "Test User" - val password = "Test Password" - - def publicRoutes: HttpRoutes[IO] = HttpRoutes.of[IO] { case GET -> Root => - Ok("Hello World") - } - - val authedRoutes: AuthedRoutes[String, IO] = AuthedRoutes.of[String, IO] { - case GET -> Root as user => Ok(user) - case req as _ => Response.notFoundFor(req) - } - - def validatePassword(creds: BasicCredentials): IO[Option[String]] = - IO.pure { - if (creds.username == username && creds.password == password) - Some(creds.username) - else None - } - - val basicAuthMiddleware: AuthMiddleware[IO, String] = - BasicAuth(realm, validatePassword) - - // - // test server - // - { - val basicAuthedService = basicAuthMiddleware(authedRoutes) - - def routes: HttpRoutes[IO] = Router[IO]( - "/pub" -> publicRoutes, - "/auth" -> basicAuthedService - ) - - test("public Route needs no authentication") { - routes(Request[IO](uri = uri"/pub/")).map { (res: Response[IO]) => - assertEquals(res.status, Status.Ok) - } - } - - test( - "BasicAuthentication should respond to a request with unknown username with 401" - ) { - val req = Request[IO]( - uri = uri"/auth", - headers = Headers(Authorization(BasicCredentials("Wrong User", password))) - ) - routes(req).foldF(IO(fail("no route"))) { (res: Response[IO]) => - IO(assertEquals(res.status, Status.Unauthorized)) >> - IO( - assertEquals( - res.headers.get[`WWW-Authenticate`].map(_.value), - Some( - Challenge("Basic", realm, Map("charset" -> "UTF-8")).toString - ) - ) - ) - } - } - - test( - "BasicAuthentication should fail to respond to a request for non existent resource" - ) { - val req = Request[IO]( - uri = uri"/doesNotExist", - headers = Headers(Authorization(BasicCredentials("Wrong User", password))) - ) - routes(req).foldF(IO(assert(true, "route does not exist"))) { (res: Response[IO]) => - IO(fail("route does not exist")) - } - } - - test( - "BasicAuthentication should respond to a request with correct credentials" - ) { - val req = Request[IO]( - uri = uri"/auth", - headers = Headers(Authorization(BasicCredentials(username, password))) - ) - routes(req).foldF(IO(fail("no route"))) { (res: Response[IO]) => - IO(assertEquals(res.status, Status.Ok)) >> - res.as[String].map(s => assertEquals(s, username)) - } - } - - test( - "BasicAuthentication responds to authenticated non-existent resource with 404" - ) { - val req = Request[IO]( - uri = uri"/auth/nonExistent", - headers = Headers(Authorization(BasicCredentials(username, password))) - ) - routes(req).foldF(IO(fail("no route"))) { (res: Response[IO]) => - IO(assertEquals(res.status, Status.NotFound)) - } - } - - // test with client now - val defaultClient: Client[IO] = Client.fromHttpApp(routes.orNotFound) - given UriConfig = UriConfig.default - import org.w3.banana.jena.JenaRdf.ops - import org.w3.banana.io.{JsonLd, RDFXML, RelRDFReader, Turtle} - import org.w3.banana.jena.io.JenaRDFReader.given - given rdfDecoders: RDFDecoders[IO, Jena] = new run.cosy.ld.http4s.RDFDecoders[IO, Jena] - // val logedClient: Client[IO] = ResponseLogger[IO](true, true, logAction = Some(s => IO(println(s))))(defaultClient) - val wallet1 = new BasicWallet[IO, Jena]( - Map( - lemonlabs.uri.Authority("localhost") -> - BasicId(username, password) - ) - )(defaultClient) - val client: Client[IO] = AuthNClient[IO](wallet1)(defaultClient) - val wallet2 = BasicWallet[IO, Jena]( - Map( - lemonlabs.uri.Authority("localhost") -> BasicId( - username, - password + "bad" - ) - ) - )(client) - val clientBad: Client[IO] = AuthNClient[IO](wallet2)(defaultClient) - - test("Wallet Based Auth") { - client.get(uri"http://localhost/auth") { (res: Response[IO]) => - IO(assertEquals(res.status, Status.Ok)) >> - res.as[String].map(s => assertEquals(s, username)) - - } - } - - test("Wallet Based Auth on Non Existent resource") { - client.get(uri"http://localhost/auth") { (res: Response[IO]) => - IO(assertEquals(res.status, Status.Ok)) >> - res.as[String].map(s => assertEquals(s, username)) - - } - } - - test("Wallet Based Auth with bad password fails on protected resources") { - clientBad.get(uri"http://localhost/auth") { (res: Response[IO]) => - IO(assertEquals(res.status, Status.Unauthorized)) - } - clientBad.get(uri"http://localhost/auth/NonExistent") { (res: Response[IO]) => - IO(assertEquals(res.status, Status.Unauthorized)) - } - } - - test("Wallet Based Auth with bad password succeeds on public resources") { - clientBad.get(uri"http://localhost/pub") { (res: Response[IO]) => - IO(assertEquals(res.status, Status.Ok)) >> - res.as[String].map(s => assertEquals(s, "Hello World")) - } - } - } - -} +class AuthNClientTest extends munit.CatsEffectSuite: + + val realm = "Test Realm" + val username = "Test User" + val password = "Test Password" + + def publicRoutes: HttpRoutes[IO] = HttpRoutes.of[IO] { case GET -> Root => Ok("Hello World") } + + val authedRoutes: AuthedRoutes[String, IO] = AuthedRoutes.of[String, IO] { + case GET -> Root as user => Ok(user) + case req as _ => Response.notFoundFor(req) + } + + def validatePassword(creds: BasicCredentials): IO[Option[String]] = IO.pure { + if creds.username == username && creds.password == password then Some(creds.username) + else None + } + + val basicAuthMiddleware: AuthMiddleware[IO, String] = BasicAuth(realm, validatePassword) + + // + // test server + // + { + val basicAuthedService = basicAuthMiddleware(authedRoutes) + + def routes: HttpRoutes[IO] = Router[IO]( + "/pub" -> publicRoutes, + "/auth" -> basicAuthedService + ) + + test("public Route needs no authentication") { + routes(Request[IO](uri = uri"/pub/")).map { (res: Response[IO]) => + assertEquals(res.status, Status.Ok) + } + } + + test( + "BasicAuthentication should respond to a request with unknown username with 401" + ) { + val req = Request[IO]( + uri = uri"/auth", + headers = Headers(Authorization(BasicCredentials("Wrong User", password))) + ) + routes(req).foldF(IO(fail("no route"))) { (res: Response[IO]) => + IO(assertEquals(res.status, Status.Unauthorized)) >> + IO( + assertEquals( + res.headers.get[`WWW-Authenticate`].map(_.value), + Some( + Challenge("Basic", realm, Map("charset" -> "UTF-8")).toString + ) + ) + ) + } + } + + test( + "BasicAuthentication should fail to respond to a request for non existent resource" + ) { + val req = Request[IO]( + uri = uri"/doesNotExist", + headers = Headers(Authorization(BasicCredentials("Wrong User", password))) + ) + routes(req).foldF(IO(assert(true, "route does not exist"))) { (res: Response[IO]) => + IO(fail("route does not exist")) + } + } + + test( + "BasicAuthentication should respond to a request with correct credentials" + ) { + val req = Request[IO]( + uri = uri"/auth", + headers = Headers(Authorization(BasicCredentials(username, password))) + ) + routes(req).foldF(IO(fail("no route"))) { (res: Response[IO]) => + IO(assertEquals(res.status, Status.Ok)) >> + res.as[String].map(s => assertEquals(s, username)) + } + } + + test( + "BasicAuthentication responds to authenticated non-existent resource with 404" + ) { + val req = Request[IO]( + uri = uri"/auth/nonExistent", + headers = Headers(Authorization(BasicCredentials(username, password))) + ) + routes(req).foldF(IO(fail("no route"))) { (res: Response[IO]) => + IO(assertEquals(res.status, Status.NotFound)) + } + } + + // test with client now + val defaultClient: Client[IO] = Client.fromHttpApp(routes.orNotFound) + given UriConfig = UriConfig.default + import org.w3.banana.jena.JenaRdf.ops + import org.w3.banana.io.{JsonLd, RDFXML, RelRDFReader, Turtle} + import org.w3.banana.jena.io.JenaRDFReader.given + given rdfDecoders: RDFDecoders[IO, Jena] = new run.cosy.ld.http4s.RDFDecoders[IO, Jena] + // val logedClient: Client[IO] = ResponseLogger[IO](true, true, logAction = Some(s => IO(println(s))))(defaultClient) + val wallet1 = new BasicWallet[IO, Jena]( + Map( + lemonlabs.uri.Authority("localhost") -> + BasicId(username, password) + ) + )(defaultClient) + val client: Client[IO] = AuthNClient[IO](wallet1)(defaultClient) + val wallet2 = BasicWallet[IO, Jena]( + Map( + lemonlabs.uri.Authority("localhost") -> BasicId( + username, + password + "bad" + ) + ) + )(client) + val clientBad: Client[IO] = AuthNClient[IO](wallet2)(defaultClient) + + test("Wallet Based Auth") { + client.get(uri"http://localhost/auth") { (res: Response[IO]) => + IO(assertEquals(res.status, Status.Ok)) >> + res.as[String].map(s => assertEquals(s, username)) + + } + } + + test("Wallet Based Auth on Non Existent resource") { + client.get(uri"http://localhost/auth") { (res: Response[IO]) => + IO(assertEquals(res.status, Status.Ok)) >> + res.as[String].map(s => assertEquals(s, username)) + + } + } + + test("Wallet Based Auth with bad password fails on protected resources") { + clientBad.get(uri"http://localhost/auth") { (res: Response[IO]) => + IO(assertEquals(res.status, Status.Unauthorized)) + } + clientBad.get(uri"http://localhost/auth/NonExistent") { (res: Response[IO]) => + IO(assertEquals(res.status, Status.Unauthorized)) + } + } + + test("Wallet Based Auth with bad password succeeds on public resources") { + clientBad.get(uri"http://localhost/pub") { (res: Response[IO]) => + IO(assertEquals(res.status, Status.Ok)) >> + res.as[String].map(s => assertEquals(s, "Hello World")) + } + } + } diff --git a/build.sbt b/build.sbt index c68cd18..2d4dc05 100644 --- a/build.sbt +++ b/build.sbt @@ -1,10 +1,10 @@ import sbt.ThisBuild import sbtcrossproject.CrossPlugin.autoImport.{crossProject, CrossType} -import Dependencies._ +import Dependencies.* name := "SolidApp" ThisBuild / organization := "net.bblfish" -ThisBuild / version := "0.1" +ThisBuild / version := "0.2" ThisBuild / scalaVersion := Ver.scala ThisBuild / startYear := Some(2021) ThisBuild / developers := List( @@ -15,8 +15,9 @@ ThisBuild / developers := List( lazy val commonSettings = Seq( name := "Solid App", - version := "0.1-SNAPSHOT", + version := "0.2-SNAPSHOT", description := "Solid App", + headerLicense := Some(HeaderLicense.ALv2("2021", "bblfish.net")), startYear := Some(2021), scalaVersion := Ver.scala, updateOptions := updateOptions.value.withCachedResolution( @@ -55,10 +56,7 @@ resolvers += "jitpack" at "https://jitpack.io" //) lazy val authN = crossProject(JVMPlatform) // , JSPlatform) - .crossType(CrossType.Full) - .in(file("authn")) - .settings(commonSettings: _*) - .settings( + .crossType(CrossType.Full).in(file("authn")).settings(commonSettings*).settings( name := "AuthNClient", description := "Http Client Middleware that knows how to use a Wallet interface to authenticate", // scalacOptions := scala3jsOptions, @@ -84,14 +82,10 @@ lazy val authN = crossProject(JVMPlatform) // , JSPlatform) // do I also need to run `npm install n3` ? // Compile / npmDependencies += NPM.n3, // Test / npmDependencies += NPM.n3 - ) - .dependsOn(wallet) + ).dependsOn(wallet) lazy val free = crossProject(JVMPlatform) // , JSPlatform) - .crossType(CrossType.Full) - .in(file("free")) - .settings(commonSettings: _*) - .settings( + .crossType(CrossType.Full).in(file("free")).settings(commonSettings*).settings( name := "Free LDP", description := "Free LDP client and provisionally interpreters (to be moved)", libraryDependencies ++= Seq( @@ -102,12 +96,8 @@ lazy val free = crossProject(JVMPlatform) // , JSPlatform) ) // an LDES client -lazy val ldes = crossProject(JVMPlatform) - .crossType(CrossType.Full) - .in(file("ldes")) - .dependsOn(ioExt4s) - .settings(commonSettings: _*) - .settings( +lazy val ldes = crossProject(JVMPlatform).crossType(CrossType.Full).in(file("ldes")) + .dependsOn(ioExt4s).settings(commonSettings*).settings( name := "LDES Client", description := "Linked Data Event Stream Libraries and Client", // scalacOptions := scala3jsOptions, @@ -124,31 +114,26 @@ lazy val ldes = crossProject(JVMPlatform) ) // make a new project called cache -lazy val cache = crossProject(JVMPlatform) - .crossType(CrossType.Full) - .in(file("cache")) - .settings(commonSettings: _*) - .settings( +lazy val cache = crossProject(JVMPlatform).crossType(CrossType.Full).in(file("cache")) + .settings(commonSettings*).settings( name := "Cache", description := "Cache", libraryDependencies ++= Seq( cats.core.value, cats.free.value, http4s.core.value, + http4s.client.value, mules.core.value ), libraryDependencies ++= Seq( // munit.value % Test, cats.munitEffect.value % Test, + http4s.theDsl.value % Test ) ) - -lazy val test = crossProject(JVMPlatform) - .crossType(CrossType.Full) - .in(file("test")) - .settings(commonSettings: _*) - .settings( +lazy val test = crossProject(JVMPlatform).crossType(CrossType.Full).in(file("test")) + .settings(commonSettings*).settings( name := "test", description := "test stuff", libraryDependencies ++= Seq( @@ -163,11 +148,8 @@ lazy val test = crossProject(JVMPlatform) ) // todo: should be moved closer to banana-rdf repo -lazy val ioExt4s = crossProject(JVMPlatform) - .crossType(CrossType.Full) - .in(file("ioExt4s")) - .settings(commonSettings: _*) - .settings( +lazy val ioExt4s = crossProject(JVMPlatform).crossType(CrossType.Full).in(file("ioExt4s")) + .settings(commonSettings*).settings( name := "IO http4s ext", description := "rdf io extensions for http4s", // scalacOptions := scala3jsOptions, @@ -185,10 +167,7 @@ lazy val ioExt4s = crossProject(JVMPlatform) //todo: we should split the wallet into client-wallet and the full wallet library // as clients of the wallet only need a minimal interface lazy val wallet = crossProject(JVMPlatform) // , JSPlatform) - .crossType(CrossType.Full) - .in(file("wallet")) - .dependsOn(ioExt4s) - .settings(commonSettings: _*) + .crossType(CrossType.Full).in(file("wallet")).dependsOn(ioExt4s).settings(commonSettings*) .settings( name := "Solid Wallet", description := "Solid Wallet libraries ", @@ -223,8 +202,7 @@ lazy val wallet = crossProject(JVMPlatform) // , JSPlatform) // Test / npmDependencies += NPM.n3 ) -lazy val scripts = crossProject(JVMPlatform) - .in(file("scripts")) +lazy val scripts = crossProject(JVMPlatform).in(file("scripts")) // .settings( // libraryDependencies ++= Seq( // other.scalaUri.value, @@ -232,8 +210,7 @@ lazy val scripts = crossProject(JVMPlatform) // crypto.bobcats.value classifier ("tests-sources") // bobcats test examples soources, // ) // ) - .dependsOn(wallet, authN, ldes) - .jvmSettings( + .dependsOn(wallet, authN, ldes).jvmSettings( libraryDependencies ++= Seq( crypto.bobcats.value classifier ("tests"), // bobcats test examples, crypto.bobcats.value classifier ("tests-sources"), // bobcats test examples soources, diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala index 426dd19..602deb6 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala @@ -1,35 +1,48 @@ +/* + * Copyright 2021 bblfish.net + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.chrisdavenport.mules.http4s -import cats._ -import cats.effect._ -import cats.implicits._ +import cats.* +import cats.effect.* +import cats.implicits.* import org.http4s.HttpDate -/** - * Cache Items are what we place in the cache, this is exposed - * so that caches can be constructed by the user for this type - **/ -final case class CacheItem( - created: HttpDate, - expires: Option[HttpDate], - response: CachedResponse, +/** Cache Items are what we place in the cache, this is exposed so that caches can be constructed by + * the user for this type + */ +final case class CacheItem[T]( + created: HttpDate, + expires: Option[HttpDate], + response: CachedResponse[T] ) -object CacheItem { +object CacheItem: - def create[F[_]: Clock: MonadThrow](response: CachedResponse, expires: Option[HttpDate]): F[CacheItem] = - HttpDate.current[F].map(date => - new CacheItem(date, expires, response) - ) + def create[F[_]: Clock: MonadThrow, T]( + response: CachedResponse[T], + expires: Option[HttpDate] + ): F[CacheItem[T]] = HttpDate.current[F].map(date => new CacheItem(date, expires, response)) - private[http4s] final case class Age(val deltaSeconds: Long) extends AnyVal - private[http4s] object Age { - def of(created: HttpDate, now: HttpDate): Age = new Age(now.epochSecond - created.epochSecond) - } - private[http4s] final case class CacheLifetime(val deltaSeconds: Long) extends AnyVal - private[http4s] object CacheLifetime { - def of(expires: Option[HttpDate], now: HttpDate): Option[CacheLifetime] = expires.map{expiredAt => - new CacheLifetime(expiredAt.epochSecond - now.epochSecond) - } - } -} \ No newline at end of file + private[http4s] final case class Age(val deltaSeconds: Long) extends AnyVal + private[http4s] object Age: + def of(created: HttpDate, now: HttpDate): Age = new Age(now.epochSecond - created.epochSecond) + private[http4s] final case class CacheLifetime(val deltaSeconds: Long) extends AnyVal + private[http4s] object CacheLifetime: + def of(expires: Option[HttpDate], now: HttpDate): Option[CacheLifetime] = expires + .map { expiredAt => + new CacheLifetime(expiredAt.epochSecond - now.epochSecond) + } diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheType.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheType.scala index 9c69ce5..53b9b34 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheType.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheType.scala @@ -1,23 +1,31 @@ +/* + * Copyright 2021 bblfish.net + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.chrisdavenport.mules.http4s -/** - * CacheTypes are in 2 flavors, private caches which are specifically - * for a single user, or public caches which can be used for multiple - * users. Private caches can cache information set to Cache-Control: private, - * whereas public caches are not allowed to cache that information - **/ -sealed trait CacheType { - /** - * Whether or not a Cache is Shared, - * public caches are shared, private caches - * are not - **/ - def isShared: Boolean = this match { +/** CacheTypes are in 2 flavors, private caches which are specifically for a single user, or public + * caches which can be used for multiple users. Private caches can cache information set to + * Cache-Control: private, whereas public caches are not allowed to cache that information + */ +sealed trait CacheType: + /** Whether or not a Cache is Shared, public caches are shared, private caches are not + */ + def isShared: Boolean = this match case CacheType.Private => false case CacheType.Public => true - } -} -object CacheType { - case object Public extends CacheType - case object Private extends CacheType -} \ No newline at end of file +object CacheType: + case object Public extends CacheType + case object Private extends CacheType diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala index 59f9132..d1ce5ab 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala @@ -1,48 +1,61 @@ +/* + * Copyright 2021 bblfish.net + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.chrisdavenport.mules.http4s +import cats.* +import cats.implicits.* +import fs2.* +import org.http4s.* import org.typelevel.vault.Vault -import org.http4s._ -import fs2._ - -import cats._ -import cats.implicits._ import scodec.bits.ByteVector // As attributes can be unbound. We cannot cache them as they may not be safe to do so. -final case class CachedResponse( - status: Status, - httpVersion: HttpVersion, - headers: Headers, - body: ByteVector -){ - def withHeaders(headers: Headers): CachedResponse = new CachedResponse( - this.status, - this.httpVersion, - headers, - this.body - ) - def toResponse[F[_]]: Response[F] = CachedResponse.toResponse(this) -} +final case class CachedResponse[T]( + status: Status, + httpVersion: HttpVersion, + headers: Headers, + body: Option[T] +): + def withHeaders(headers: Headers): CachedResponse[T] = new CachedResponse[T]( + this.status, + this.httpVersion, + headers, + this.body + ) + +object CachedResponse: -object CachedResponse { + def errorResponse[T](status: Status): CachedResponse[T] = new CachedResponse[T]( + status, + HttpVersion.`HTTP/1.1`, + Headers.empty, + None + ) - def fromResponse[F[_], G[_]: Functor](response: Response[F])(implicit compiler: Compiler[F,G]): G[CachedResponse] = { - response.body.compile.to(ByteVector).map{bv => - new CachedResponse( - response.status, - response.httpVersion, - response.headers, - bv - ) - } - } + def fromResponse[F[_], G[_]: Functor]( + response: Response[F] + )(using compiler: Compiler[F, G]): G[CachedResponse[ByteVector]] = response.body.compile + .to(ByteVector).map { bv => + new CachedResponse[ByteVector]( + response.status, + response.httpVersion, + response.headers, + Some(bv) + ) + } - def toResponse[F[_]](cachedResponse: CachedResponse): Response[F] = - Response( - cachedResponse.status, - cachedResponse.httpVersion, - cachedResponse.headers, - Stream.chunk(Chunk.byteVector(cachedResponse.body)), - Vault.empty - ) -} \ No newline at end of file +end CachedResponse diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/CacheRules.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/CacheRules.scala index 5e5191f..862a5c6 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/CacheRules.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/CacheRules.scala @@ -1,236 +1,225 @@ +/* + * Copyright 2021 bblfish.net + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.chrisdavenport.mules.http4s.internal -import io.chrisdavenport.mules.http4s._ - -import org.http4s._ -import org.http4s.headers._ -import org.http4s.CacheDirective._ -import scala.concurrent.duration._ -import cats._ -import cats.implicits._ -import cats.data._ -import org.typelevel.ci._ - -private[http4s] object CacheRules { - - def requestCanUseCached[F[_]](req: Request[F]): Boolean = - methodIsCacheable(req.method) && - !req.headers.get[`Cache-Control`].exists{ - _.values.exists{ - case `no-cache`(_) => true - case _ => false - } - } - - private val cacheableMethods: Set[Method] = Set( - Method.GET, - Method.HEAD, - // Method.POST // Eventually make this work. - ) - - def methodIsCacheable(m: Method): Boolean = cacheableMethods.contains(m) - - private val cacheableStatus: Set[Status] = Set( - Status.Ok, // 200 - Status.NonAuthoritativeInformation, // 203 - Status.NoContent, // 204 - Status.PartialContent, // 206 - Status.MultipleChoices, // 300 - Status.MovedPermanently, // 301 - // Status.NotModified , // 304 - Status.NotFound, // 404 - Status.MethodNotAllowed, // 405 - Status.Gone, // 410 - Status.UriTooLong, // 414 - Status.NotImplemented, // 501 - ) - - def statusIsCacheable(s: Status): Boolean = cacheableStatus.contains(s) - - def cacheAgeAcceptable[F[_]](req: Request[F], item: CacheItem, now: HttpDate): Boolean = { - req.headers.get[`Cache-Control`] match { - case None => - +import io.chrisdavenport.mules.http4s.* + +import org.http4s.* +import org.http4s.headers.* +import org.http4s.CacheDirective.* +import scala.concurrent.duration.* +import cats.* +import cats.implicits.* +import cats.data.* +import org.typelevel.ci.* + +private[http4s] object CacheRules: + + def requestCanUseCached[F[_]](req: Request[F]): Boolean = methodIsCacheable(req.method) && + !req.headers.get[`Cache-Control`].exists { + _.values.exists { + case `no-cache`(_) => true + case _ => false + } + } + + private val cacheableMethods: Set[Method] = Set( + Method.GET, + Method.HEAD + // Method.POST // Eventually make this work. + ) + + def methodIsCacheable(m: Method): Boolean = cacheableMethods.contains(m) + + private val cacheableStatus: Set[Status] = Set( + Status.Ok, // 200 + Status.NonAuthoritativeInformation, // 203 + Status.NoContent, // 204 + Status.PartialContent, // 206 + Status.MultipleChoices, // 300 + Status.MovedPermanently, // 301 + // Status.NotModified , // 304 + Status.NotFound, // 404 + Status.MethodNotAllowed, // 405 + Status.Gone, // 410 + Status.UriTooLong, // 414 + Status.NotImplemented // 501 + ) + + def statusIsCacheable(s: Status): Boolean = cacheableStatus.contains(s) + + def cacheAgeAcceptable[F[_]](req: Request[F], item: CacheItem[?], now: HttpDate): Boolean = + req.headers.get[`Cache-Control`] match + case None => // TODO: Investigate how this check works with cache-control // If the data in the cache is expired and client does not explicitly // accept stale data, then age is not ok. item.expires.map(expiresAt => expiresAt >= now).getOrElse(true) - case Some(`Cache-Control`(values)) => + case Some(`Cache-Control`(values)) => val age = CacheItem.Age.of(item.created, now) val lifetime = CacheItem.CacheLifetime.of(item.expires, now) val maxAgeMet: Boolean = values.toList - .collectFirst{ case c@CacheDirective.`max-age`(_) => c } - .map(maxAge => age.deltaSeconds.seconds <= maxAge.deltaSeconds ) - .getOrElse(true) + .collectFirst { case c @ CacheDirective.`max-age`(_) => c } + .map(maxAge => age.deltaSeconds.seconds <= maxAge.deltaSeconds).getOrElse(true) val maxStaleMet: Boolean = { - for { - maxStale <- values.toList.collectFirst{ case c@CacheDirective.`max-stale`(_) => c.deltaSeconds}.flatten - stale <- lifetime - } yield if (stale.deltaSeconds >= 0) true else stale.deltaSeconds.seconds <= maxStale + for + maxStale <- values.toList.collectFirst { case c @ CacheDirective.`max-stale`(_) => + c.deltaSeconds + }.flatten + stale <- lifetime + yield if stale.deltaSeconds >= 0 then true else stale.deltaSeconds.seconds <= maxStale }.getOrElse(true) val minFreshMet: Boolean = { - for { - minFresh <- values.toList.collectFirst{case CacheDirective.`min-fresh`(seconds) => seconds} - expiresAt <- item.expires - } yield (expiresAt.epochSecond - now.epochSecond).seconds <= minFresh + for + minFresh <- values.toList.collectFirst { case CacheDirective.`min-fresh`(seconds) => + seconds + } + expiresAt <- item.expires + yield (expiresAt.epochSecond - now.epochSecond).seconds <= minFresh }.getOrElse(true) - + // println(s"Age- $age, Lifetime- $lifetime, maxAgeMet: $maxAgeMet, maxStaleMet: $maxStaleMet, minFreshMet: $minFreshMet") - - maxAgeMet && maxStaleMet && minFreshMet - } - } - - def onlyIfCached[F[_]](req: Request[F]): Boolean = req.headers.get[`Cache-Control`] - .exists{_.values.exists{ - case `only-if-cached` => true - case _ => false - }} - - def cacheControlNoStoreExists[F[_]](response: Response[F]): Boolean = response.headers - .get[`Cache-Control`] - .toList - .flatMap(_.values.toList) - .exists{ - case CacheDirective.`no-store` => true - case _ => false - } - - def cacheControlPrivateExists[F[_]](response: Response[F]): Boolean = response.headers - .get[`Cache-Control`] - .toList - .flatMap(_.values.toList) - .exists{ - case CacheDirective.`private`(_) => true - case _ => false - } - - def authorizationHeaderExists[F[_]](response: Response[F]): Boolean = response.headers - .get[Authorization] - .isDefined - - def cacheControlPublicExists[F[_]](response: Response[F]): Boolean = response.headers - .get[`Cache-Control`] - .toList - .flatMap(_.values.toList) - .exists{ - case CacheDirective.public => true - case _ => false - } - - def mustRevalidate[F[_]](response: Message[F]): Boolean = { - response.headers.get[`Cache-Control`].exists{_.values.exists{ - case CacheDirective.`no-cache`(_) => true - case CacheDirective.`max-age`(age) if age <= 0.seconds => true - case _ => false - }} || response.headers.get(CIString("Pragma")).exists(_.exists(_.value === "no-cache")) - } - - def isCacheable[F[_]](req: Request[F], response: Response[F], cacheType: CacheType): Boolean = { - if (!cacheableMethods.contains(req.method)) { - // println(s"Request Method ${req.method} - not Cacheable") - false - } else if (!statusIsCacheable(response.status)) { - // println(s"Response Status ${response.status} - not Cacheable") - false - } else if (cacheControlNoStoreExists(response)) { - // println("Cache-Control No-Store is present - not Cacheable") - false - } else if (cacheType.isShared && cacheControlPrivateExists(response)) { - // println("Cache is shared and Cache-Control private exists - not Cacheable") - false - } else if (cacheType.isShared && response.headers.get(CIString("Vary")).exists(h => h.exists(_.value === "*"))) { - // println("Cache is shared and Vary header exists as * - not Cacheable") - false - } else if (cacheType.isShared && authorizationHeaderExists(response) && !cacheControlPublicExists(response)) { - // println("Cache is Shared and Authorization Header is present and Cache-Control public is not present - not Cacheable") - false - } else if (mustRevalidate(response) && !(response.headers.get[ETag].isDefined || response.headers.get[`Last-Modified`].isDefined)) { - false - } else if (req.method === Method.GET || req.method === Method.HEAD) { - true - } else if (cacheControlPublicExists(response) || cacheControlPrivateExists(response)) { - true - } else { - response.headers.get[Expires].isDefined - } - } - - def shouldInvalidate[F[_]](request: Request[F], response: Response[F]): Boolean = { - if (Set(Status.NotFound, Status.Gone).contains(response.status)) { - true - } else if (Set(Method.GET, Method.HEAD: Method).contains(request.method)){ - false - } else response.status.isSuccess - } - - def getIfMatch(cachedResponse: CachedResponse): Option[`If-None-Match`] = - cachedResponse.headers.get[ETag].map(_.tag).flatMap{etag => - if (etag.weakness != EntityTag.Weak) `If-None-Match`(NonEmptyList.of(etag).some).some - else None - } - - def getIfUnmodifiedSince(cachedResponse: CachedResponse): Option[`If-Unmodified-Since`] = { - for { - lastModified <- cachedResponse.headers.get[`Last-Modified`] - date <- cachedResponse.headers.get[Date] - _ <- Alternative[Option].guard(date.date.epochSecond - lastModified.date.epochSecond >= 60L) - } yield `If-Unmodified-Since`(lastModified.date) - } - - object FreshnessAndExpiration { - // Age in Seconds - private def getAge[F[_]](now: HttpDate, response: Message[F]): FiniteDuration = { - - // Age Or Zero - val initAgeSeconds: Long = now.epochSecond - - response.headers.get[Date].map(date => date.date.epochSecond) - .getOrElse(0L) - - response.headers.get[Age] - .map(age => Math.max(age.age, initAgeSeconds)) - .getOrElse(initAgeSeconds) - .seconds - } - - // Since We do not emit warnings on cache times over 24 hours, limit cache time - // to max of 24 hours. - private def freshnessLifetime[F[_]](now: HttpDate, response: Message[F]) = { - response.headers.get[`Cache-Control`] - .flatMap{ - case `Cache-Control`(directives) => - directives.collectFirst{ - case `max-age`(deltaSeconds) => - deltaSeconds match { - case finite: FiniteDuration => finite - case _ => 24.hours - } - } - }.orElse{ - for { - exp <- response.headers.get[Expires] - date <- response.headers.get[Date] - } yield (exp.expirationDate.epochSecond - date.date.epochSecond).seconds - }.orElse{ - response.headers.get[`Last-Modified`] - .map{lm => - val estimatedLifetime = (now.epochSecond - lm.date.epochSecond) / 10 - Math.min(24.hours.toSeconds, estimatedLifetime).seconds - } - }.getOrElse(24.hours) - } + maxAgeMet && maxStaleMet && minFreshMet - def getExpires[F[_]](now: HttpDate, response: Message[F]): HttpDate = { - val age = getAge(now, response) - val lifetime = freshnessLifetime(now, response) - val ttl = lifetime - age + def onlyIfCached[F[_]](req: Request[F]): Boolean = req.headers.get[`Cache-Control`].exists { + _.values.exists { + case `only-if-cached` => true + case _ => false + } + } + + def cacheControlNoStoreExists[F[_]](response: Response[F]): Boolean = response.headers + .get[`Cache-Control`].toList.flatMap(_.values.toList).exists { + case CacheDirective.`no-store` => true + case _ => false + } + + def cacheControlPrivateExists[F[_]](response: Response[F]): Boolean = response.headers + .get[`Cache-Control`].toList.flatMap(_.values.toList).exists { + case CacheDirective.`private`(_) => true + case _ => false + } + + def authorizationHeaderExists[F[_]](response: Response[F]): Boolean = response.headers + .get[Authorization].isDefined + + def cacheControlPublicExists[F[_]](response: Response[F]): Boolean = response.headers + .get[`Cache-Control`].toList.flatMap(_.values.toList).exists { + case CacheDirective.public => true + case _ => false + } + + def mustRevalidate[F[_]](response: Message[F]): Boolean = response.headers.get[`Cache-Control`] + .exists { + _.values.exists { + case CacheDirective.`no-cache`(_) => true + case CacheDirective.`max-age`(age) if age <= 0.seconds => true + case _ => false + } + } || response.headers.get(CIString("Pragma")).exists(_.exists(_.value === "no-cache")) + + def isCacheable[F[_]](req: Request[F], response: Response[F], cacheType: CacheType): Boolean = + if !cacheableMethods.contains(req.method) then + // println(s"Request Method ${req.method} - not Cacheable") + false + else if !statusIsCacheable(response.status) then + // println(s"Response Status ${response.status} - not Cacheable") + false + else if cacheControlNoStoreExists(response) then + // println("Cache-Control No-Store is present - not Cacheable") + false + else if cacheType.isShared && cacheControlPrivateExists(response) then + // println("Cache is shared and Cache-Control private exists - not Cacheable") + false + else if cacheType.isShared && response.headers.get(CIString("Vary")) + .exists(h => h.exists(_.value === "*")) + then + // println("Cache is shared and Vary header exists as * - not Cacheable") + false + else if cacheType.isShared && authorizationHeaderExists(response) && !cacheControlPublicExists( + response + ) + then + // println("Cache is Shared and Authorization Header is present and Cache-Control public is not present - not Cacheable") + false + else if mustRevalidate(response) && !(response.headers.get[ETag].isDefined || response.headers + .get[`Last-Modified`].isDefined) + then false + else if req.method === Method.GET || req.method === Method.HEAD then true + else if cacheControlPublicExists(response) || cacheControlPrivateExists(response) then true + else response.headers.get[Expires].isDefined + + def shouldInvalidate[F[_]](request: Request[F], response: Response[F]): Boolean = + if Set(Status.NotFound, Status.Gone).contains(response.status) then true + else if Set(Method.GET, Method.HEAD: Method).contains(request.method) then false + else response.status.isSuccess + + def getIfMatch(cachedResponse: CachedResponse[?]): Option[`If-None-Match`] = cachedResponse + .headers.get[ETag].map(_.tag).flatMap { etag => + if etag.weakness != EntityTag.Weak then `If-None-Match`(NonEmptyList.of(etag).some).some + else None + } + + def getIfUnmodifiedSince(cachedResponse: CachedResponse[?]): Option[`If-Unmodified-Since`] = + for + lastModified <- cachedResponse.headers.get[`Last-Modified`] + date <- cachedResponse.headers.get[Date] + _ <- Alternative[Option].guard(date.date.epochSecond - lastModified.date.epochSecond >= 60L) + yield `If-Unmodified-Since`(lastModified.date) + + object FreshnessAndExpiration: + // Age in Seconds + private def getAge[F[_]](now: HttpDate, response: Message[F]): FiniteDuration = + + // Age Or Zero + val initAgeSeconds: Long = now.epochSecond - + response.headers.get[Date].map(date => date.date.epochSecond).getOrElse(0L) + + response.headers.get[Age].map(age => Math.max(age.age, initAgeSeconds)) + .getOrElse(initAgeSeconds).seconds + + // Since We do not emit warnings on cache times over 24 hours, limit cache time + // to max of 24 hours. + private def freshnessLifetime[F[_]](now: HttpDate, response: Message[F]) = response.headers + .get[`Cache-Control`].flatMap { case `Cache-Control`(directives) => + directives.collectFirst { case `max-age`(deltaSeconds) => + deltaSeconds match + case finite: FiniteDuration => finite + case _ => 24.hours + } + }.orElse { + for + exp <- response.headers.get[Expires] + date <- response.headers.get[Date] + yield (exp.expirationDate.epochSecond - date.date.epochSecond).seconds + }.orElse { + response.headers.get[`Last-Modified`].map { lm => + val estimatedLifetime = (now.epochSecond - lm.date.epochSecond) / 10 + Math.min(24.hours.toSeconds, estimatedLifetime).seconds + } + }.getOrElse(24.hours) - HttpDate.unsafeFromEpochSecond(now.epochSecond + ttl.toSeconds) - } - } + def getExpires[F[_]](now: HttpDate, response: Message[F]): HttpDate = + val age = getAge(now, response) + val lifetime = freshnessLifetime(now, response) + val ttl = lifetime - age -} \ No newline at end of file + HttpDate.unsafeFromEpochSecond(now.epochSecond + ttl.toSeconds) diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala index fdb3ad2..e314f66 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala @@ -1,76 +1,91 @@ +/* + * Copyright 2021 bblfish.net + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.chrisdavenport.mules.http4s.internal -import io.chrisdavenport.mules.http4s._ -import org.http4s._ -import io.chrisdavenport.mules._ -import cats._ -import cats.syntax.all._ -import cats.data._ -import cats.effect._ +import cats.* +import cats.data.* +import cats.effect.* +import cats.syntax.all.* +import io.chrisdavenport.mules.* +import io.chrisdavenport.mules.http4s.* +import org.http4s.* import org.http4s.Header.ToRaw.modelledHeadersToRaw +import run.cosy.http.cache.TreeDirCache +import run.cosy.http.cache.ServerNotFound -private[http4s] class Caching[F[_]: Concurrent: Clock] private[http4s] (cache: Cache[F, (Method, Uri), CacheItem], cacheType: CacheType){ +case class Caching[F[_]: Concurrent: Clock, T]( + cache: TreeDirCache[F, CacheItem[T]], + interpret: Response[F] => F[CachedResponse[T]], + cacheType: CacheType +): - def request[G[_]: FlatMap](app: Kleisli[G, Request[F], Response[F]], fk: F ~> G)(req: Request[F]): G[Response[F]] = { - if (CacheRules.requestCanUseCached(req)) { - for { - cachedValue <- fk(cache.lookup((req.method, req.uri))) - now <- fk(HttpDate.current[F]) - out <- cachedValue match { - case None => - if (CacheRules.onlyIfCached(req)) fk(Response[F](Status.GatewayTimeout).pure[F]) - else { - app.run(req) - .flatMap(resp => fk(withResponse(req, resp))) - } - case Some(item) => - if (CacheRules.cacheAgeAcceptable(req, item, now)) { - fk(item.response.toResponse[F].pure[F]) - } else { - app.run( - req - .putHeaders(CacheRules.getIfMatch(item.response).map(modelledHeadersToRaw(_)).toSeq:_*) - .putHeaders(CacheRules.getIfUnmodifiedSince(item.response).map(modelledHeadersToRaw(_)).toSeq:_*) - ).flatMap(resp => fk(withResponse(req, resp))) - } - } - } yield out - } else { - app.run(req) - .flatMap(resp => fk(withResponse(req, resp))) - } - } + def request[G[_]: FlatMap]( + app: Kleisli[G, Request[F], Response[F]], + fk: F ~> G + )(req: Request[F]): G[CachedResponse[T]] = + if CacheRules.requestCanUseCached(req) + then + for + cachedValue <- fk(cache.lookup(req.uri).recover { case _: ServerNotFound => None }) + now <- fk(HttpDate.current[F]) + out <- cachedValue match + case None => + if CacheRules.onlyIfCached(req) + then fk(CachedResponse.errorResponse[T](Status.GatewayTimeout).pure[F]) + else app.run(req).flatMap(resp => fk(withResponse(req, resp))) + case Some(item) => + if CacheRules.cacheAgeAcceptable(req, item, now) then fk(item.response.pure[F]) + else + app.run( + req.putHeaders( + CacheRules.getIfMatch(item.response).map(modelledHeadersToRaw(_)).toSeq* + ).putHeaders( + CacheRules.getIfUnmodifiedSince(item.response).map(modelledHeadersToRaw(_)) + .toSeq* + ) + ).flatMap(resp => fk(withResponse(req, resp))) + yield out + else app.run(req).flatMap(resp => fk(withResponse(req, resp))) + end request - private def withResponse(req: Request[F], resp: Response[F]): F[Response[F]] = { - { - if (CacheRules.shouldInvalidate(req, resp)){ - cache.delete((req.method, req.uri)) - } else Applicative[F].unit - } *> { - if (CacheRules.isCacheable(req, resp, cacheType)){ - for { - cachedResp <- resp.status match { - case Status.NotModified => - cache.lookup((req.method, req.uri)) - .flatMap( - _.map{item => - val cached = item.response - cached.withHeaders(resp.headers ++ cached.headers).pure[F] - } - .getOrElse(CachedResponse.fromResponse[F, F](resp)) - ) - case _ => CachedResponse.fromResponse[F, F](resp) - } - now <- HttpDate.current[F] - expires = CacheRules.FreshnessAndExpiration.getExpires(now, resp) - item <- CacheItem.create(cachedResp, expires.some) - _ <- cache.insert((req.method, req.uri), item) - } yield cachedResp.toResponse[F] - - } else { - resp.pure[F] - } - } - } + private def withResponse( + req: Request[F], + resp: Response[F] + ): F[CachedResponse[T]] = { + if CacheRules.shouldInvalidate(req, resp) then cache.delete(req.uri) + else Applicative[F].unit + } *> { + if CacheRules.isCacheable(req, resp, cacheType) then + for + cachedResp <- resp.status match + case Status.NotModified => cache.lookup(req.uri).flatMap( + _.map { item => + val cached = item.response + cached.withHeaders(resp.headers ++ cached.headers).pure[F] + }.getOrElse(interpret(resp)) + ) + case _ => interpret(resp) + now <- HttpDate.current[F] + expires = CacheRules.FreshnessAndExpiration.getExpires(now, resp) + item <- CacheItem.create(cachedResp, expires.some) + _ <- cache.insert(req.uri, item) + yield cachedResp + else interpret(resp) + } + end withResponse -} \ No newline at end of file +end Caching diff --git a/cache/shared/src/main/scala/run/cosy/http/cache/Cache.scala b/cache/shared/src/main/scala/run/cosy/http/cache/Cache.scala index c247b8a..a34b951 100644 --- a/cache/shared/src/main/scala/run/cosy/http/cache/Cache.scala +++ b/cache/shared/src/main/scala/run/cosy/http/cache/Cache.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 bblfish.net + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package run.cosy.http.cache import cats.effect.kernel.{Ref, Sync} @@ -9,8 +25,8 @@ import run.cosy.http.cache.DirTree.* import run.cosy.http.cache.TreeDirCache.WebCache object TreeDirCache: - type Server = (Uri.Scheme, Uri.Authority) - type WebCache[X] = Map[Server, DirTree[Option[X]]] + type Server = (Uri.Scheme, Uri.Authority) + type WebCache[X] = Map[Server, DirTree[Option[X]]] sealed trait TreeDirException extends Exception case class ServerNotFound(uri: Uri) extends TreeDirException @@ -22,69 +38,67 @@ case class TreeDirCache[F[_], X]( )(using F: Sync[F]) extends Cache[F, Uri, X]: - /* todo: consider if that is really wise. It doe */ - override def delete(k: Uri): F[Unit] = - if k.path.segments.isEmpty && !k.path.endsWithSlash - then F.pure(()) - else - for + /* todo: consider if that is really wise. It doe */ + override def delete(k: Uri): F[Unit] = + if k.path.segments.isEmpty && !k.path.endsWithSlash + then F.pure(()) + else + for + scheme <- F.fromOption(k.scheme, IncompleteServiceInfo(k)) + auth <- F.fromOption(k.authority, IncompleteServiceInfo(k)) + server = (scheme, auth) + _ <- cacheRef.update { webCache => + webCache.get(server) match + case Some(tree) => webCache.updated(server, tree.set(k.path.segments, None)) + case None => webCache + } + yield () + + /** Deleting a server without a path, results in loosing all info in the path. todo: consider + * if that is really wise. It doe + */ + def deleteBelow(k: Uri): F[Unit] = + for scheme <- F.fromOption(k.scheme, IncompleteServiceInfo(k)) auth <- F.fromOption(k.authority, IncompleteServiceInfo(k)) server = (scheme, auth) _ <- cacheRef.update { webCache => - webCache.get(server) match - case Some(tree) => webCache.updated(server, tree.set(k.path.segments, None)) - case None => webCache + if k.path.segments.isEmpty && !k.path.endsWithSlash + then webCache - server + else + webCache.get(server).map(_.setDirAt(k.path.segments, DirTree.pure(None))) match + case Some(tree) => webCache.updated(server, tree) + case None => webCache } - yield () + yield () - /** Deleting a server without a path, results in loosing all info in the path. todo: consider if - * that is really wise. It doe - */ - def deleteBelow(k: Uri): F[Unit] = - for - scheme <- F.fromOption(k.scheme, IncompleteServiceInfo(k)) - auth <- F.fromOption(k.authority, IncompleteServiceInfo(k)) - server = (scheme, auth) - _ <- cacheRef.update { webCache => - if k.path.segments.isEmpty && !k.path.endsWithSlash - then webCache - server - else - webCache - .get(server) - .map(_.setDirAt(k.path.segments, DirTree.pure(None))) match - case Some(tree) => webCache.updated(server, tree) - case None => webCache - } - yield () - - override def insert(k: Uri, v: X): F[Unit] = - for - scheme <- F.fromOption(k.scheme, IncompleteServiceInfo(k)) - auth <- F.fromOption(k.authority, IncompleteServiceInfo(k)) - server = (scheme, auth) - _ <- cacheRef.update { webCache => - val tree = webCache.getOrElse(server, DirTree.pure(None)) - webCache.updated(server, tree.insertAt(k.path.segments, Some(v), None)) - } - yield () + override def insert(k: Uri, v: X): F[Unit] = + for + scheme <- F.fromOption(k.scheme, IncompleteServiceInfo(k)) + auth <- F.fromOption(k.authority, IncompleteServiceInfo(k)) + server = (scheme, auth) + _ <- cacheRef.update { webCache => + val tree = webCache.getOrElse(server, DirTree.pure(None)) + webCache.updated(server, tree.insertAt(k.path.segments, Some(v), None)) + } + yield () - override def lookup(k: Uri): F[Option[X]] = - for - scheme <- F.fromOption(k.scheme, IncompleteServiceInfo(k)) - auth <- F.fromOption(k.authority, IncompleteServiceInfo(k)) - webCache <- cacheRef.get - server = (scheme, auth) - tree <- F.fromOption(webCache.get(server), ServerNotFound(k)) - yield - val (path, v) = tree.find(k.path.segments) - if path.isEmpty then v else None + override def lookup(k: Uri): F[Option[X]] = + for + scheme <- F.fromOption(k.scheme, IncompleteServiceInfo(k)) + auth <- F.fromOption(k.authority, IncompleteServiceInfo(k)) + webCache <- cacheRef.get + server = (scheme, auth) + tree <- F.fromOption(webCache.get(server), ServerNotFound(k)) + yield + val (path, v) = tree.find(k.path.segments) + if path.isEmpty then v else None - def findClosest(k: Uri)(matcher: Option[X] => Boolean): F[Option[X]] = - for - scheme <- F.fromOption(k.scheme, IncompleteServiceInfo(k)) - auth <- F.fromOption(k.authority, IncompleteServiceInfo(k)) - webCache <- cacheRef.get - server = (scheme, auth) - tree <- F.fromOption(webCache.get(server), ServerNotFound(k)) - yield tree.findClosest(k.path.segments)(matcher).flatten + def findClosest(k: Uri)(matcher: Option[X] => Boolean): F[Option[X]] = + for + scheme <- F.fromOption(k.scheme, IncompleteServiceInfo(k)) + auth <- F.fromOption(k.authority, IncompleteServiceInfo(k)) + webCache <- cacheRef.get + server = (scheme, auth) + tree <- F.fromOption(webCache.get(server), ServerNotFound(k)) + yield tree.findClosest(k.path.segments)(matcher).flatten diff --git a/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala b/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala index 3e401ed..0758082 100644 --- a/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala +++ b/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 bblfish.net + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package run.cosy.http.cache import cats.Eval @@ -8,113 +24,107 @@ import scala.annotation.tailrec import scala.util.Right object DirTree: - // todo: we really need a Uri abstraction - type Dir[X] = Map[org.http4s.Uri.Path.Segment, X] - type DirTree[X] = Cofree[Dir[_], X] - type Path = Seq[Path.Segment] - - /** A Path of DirTree[R,A]s is used to take apart a DirTree[R,A] structure, in the reverse - * direction. Note the idea is that each link (name, dt) points from dt via name in the hashMap - * to the next deeper node. The APath is in reverse direction so we have List( "newName" <- dir2 - * , "dir2in1" <- dir1 , "dir1" <- rootDir ) An empty List would just refer to the root - * DirTree[R,A] - */ - type ZPath[A] = Seq[ZLink[A]] - - /** A Zip Link - * @param from - * the DirTree from which the link is pointing - * @param linkName - * the name of the link. There may be nothing at the end of that. - */ - case class ZLink[A](from: DirTree[A], linkName: Path.Segment) - - /** The context of an unzipped DirTree the first projection is either - * -Right: the object at the end of the path - * -Left: the remaining path. If it is Nil then the path is pointing into a position to which one - * can add the second is the A Path, from the object to the root so that the object can be - * reconstituted - */ - type ZipContext[A] = (Either[Path, DirTree[A]], ZPath[A]) - - def pure[X](x: X): DirTree[X] = Cofree(x, Eval.now(Map.empty)) - def apply[X](x: X, dirs: Dir[DirTree[X]]): DirTree[X] = Cofree(x, Eval.now(dirs)) - - extension [X](thizDt: DirTree[X]) - def ->(name: Path.Segment): ZLink[X] = ZLink(thizDt, name) - - /** find the closest node X available when following Path return the remaining path - */ - @tailrec - def find(at: Path): (Path, X) = - at match - case Seq() => (at, thizDt.head) - case Seq(name, tail*) => - val dir: Dir[Cofree[Dir, X]] = thizDt.tail.value - dir.get(name) match - case None => (at, thizDt.head) - case Some(tree) => tree.find(tail) - end find - - def unzipAlong(path: Path): ZipContext[X] = + // todo: we really need a Uri abstraction + type Dir[X] = Map[org.http4s.Uri.Path.Segment, X] + type DirTree[X] = Cofree[Dir, X] + type Path = Seq[Path.Segment] + + /** A Path of DirTree[R,A]s is used to take apart a DirTree[R,A] structure, in the reverse + * direction. Note the idea is that each link (name, dt) points from dt via name in the hashMap + * to the next deeper node. The APath is in reverse direction so we have List( "newName" <- dir2 + * , "dir2in1" <- dir1 , "dir1" <- rootDir ) An empty List would just refer to the root + * DirTree[R,A] + */ + type ZPath[A] = Seq[ZLink[A]] + + /** A Zip Link + * @param from + * the DirTree from which the link is pointing + * @param linkName + * the name of the link. There may be nothing at the end of that. + */ + case class ZLink[A](from: DirTree[A], linkName: Path.Segment) + + /** The context of an unzipped DirTree the first projection is either + * -Right: the object at the end of the path + * -Left: the remaining path. If it is Nil then the path is pointing into a position to which + * one can add the second is the A Path, from the object to the root so that the object can be + * reconstituted + */ + type ZipContext[A] = (Either[Path, DirTree[A]], ZPath[A]) + + def pure[X](x: X): DirTree[X] = Cofree(x, Eval.now(Map.empty)) + def apply[X](x: X, dirs: Dir[DirTree[X]]): DirTree[X] = Cofree(x, Eval.now(dirs)) + + extension [X](thizDt: DirTree[X]) + def ->(name: Path.Segment): ZLink[X] = ZLink(thizDt, name) + + /** find the closest node X available when following Path return the remaining path + */ @tailrec - def loop(dt: DirTree[X], path: Path, result: ZPath[X]): ZipContext[X] = - path match + def find(at: Path): (Path, X) = at match + case Seq() => (at, thizDt.head) + case Seq(name, tail*) => + val dir: Dir[Cofree[Dir, X]] = thizDt.tail.value + dir.get(name) match + case None => (at, thizDt.head) + case Some(tree) => tree.find(tail) + end find + + def unzipAlong(path: Path): ZipContext[X] = + @tailrec + def loop(dt: DirTree[X], path: Path, result: ZPath[X]): ZipContext[X] = path match case Seq() => (Right(dt), result) case Seq(name, rest*) => if dt.tail.value.isEmpty then (Left(rest), ZLink(dt, name) +: result) else - dt.tail.value.get(name) match + dt.tail.value.get(name) match case None => (Left(rest), ZLink(dt, name) +: result) - case Some(dtchild) => - loop(dtchild, rest, ZLink(dt, name) +: result) - end loop - - loop(thizDt, path, Seq()) - end unzipAlong - - /** find the closest node matching `select` going backwards from where we got */ - def findClosest(path: Path)(select: X => Boolean): Option[X] = - unzipAlong(path) match - case (Right(dt), zpath) => dt.head +: zpath.map(_.from.head) find select - case (Left(_), zpath) => zpath.map(_.from.head) find select - - /** Rezip along zpath */ - def rezip(path: ZPath[X]): DirTree[X] = - def loop(path: ZPath[X], dt: DirTree[X]): DirTree[X] = - path match + case Some(dtchild) => loop(dtchild, rest, ZLink(dt, name) +: result) + end loop + + loop(thizDt, path, Seq()) + end unzipAlong + + /** find the closest node matching `select` going backwards from where we got */ + def findClosest(path: Path)(select: X => Boolean): Option[X] = unzipAlong(path) match + case (Right(dt), zpath) => dt.head +: zpath.map(_.from.head) find select + case (Left(_), zpath) => zpath.map(_.from.head) find select + + /** Rezip along zpath */ + def rezip(path: ZPath[X]): DirTree[X] = + def loop(path: ZPath[X], dt: DirTree[X]): DirTree[X] = path match case Seq() => dt case Seq(ZLink(from, name), tail*) => val newTail = from.tail.value + (name -> dt) loop(tail, from.copy(tail = Eval.now(newTail))) - loop(path, thizDt) - - /** set value at path creating new directories with default values if needed */ - def insertAt(path: Path, value: X, default: X): DirTree[X] = - thizDt.unzipAlong(path) match - case (Left(path), zpath) => - pure(value).rezip(path.reverse.map(p => pure(default) -> p).appendedAll(zpath)) - case (Right(dt), zpath) => dt.copy(head = value).rezip(zpath) - - /** set value at point `path` but wihtout creating intermediary directories. This is actually - * useful for deleting an entry without affecting the environement: ie. only delete what exists - */ - def set(path: Path, value: X): DirTree[X] = - thizDt.unzipAlong(path) match - case (Right(dt), zpath) => dt.copy(head = value).rezip(zpath) - case _ => thizDt - - /** set value at point `path` but wihtout creating intermediary directories. This is actually - * useful for deleting an entry without affecting the environement: ie. only delete what exists - */ - def setDirAt(path: Path, newDt: DirTree[X]): DirTree[X] = - thizDt.unzipAlong(path) match - case (Right(_), zpath) => newDt.rezip(zpath) - case _ => thizDt - - /** set dirTree at path creating new directories with default values if needed - */ - def insertDirAt(path: Path, dt: DirTree[X], default: X): DirTree[X] = - thizDt.unzipAlong(path) match - case (Left(path), zpath) => dt.rezip(path.map(p => pure(default) -> p).appendedAll(zpath)) - case (Right(dt), zpath) => dt.rezip(zpath) + loop(path, thizDt) + + /** set value at path creating new directories with default values if needed */ + def insertAt(path: Path, value: X, default: X): DirTree[X] = thizDt.unzipAlong(path) match + case (Left(path), zpath) => pure(value) + .rezip(path.reverse.map(p => pure(default) -> p).appendedAll(zpath)) + case (Right(dt), zpath) => dt.copy(head = value).rezip(zpath) + + /** set value at point `path` but wihtout creating intermediary directories. This is actually + * useful for deleting an entry without affecting the environement: ie. only delete what + * exists + */ + def set(path: Path, value: X): DirTree[X] = thizDt.unzipAlong(path) match + case (Right(dt), zpath) => dt.copy(head = value).rezip(zpath) + case _ => thizDt + + /** set value at point `path` but wihtout creating intermediary directories. This is actually + * useful for deleting an entry without affecting the environement: ie. only delete what + * exists + */ + def setDirAt(path: Path, newDt: DirTree[X]): DirTree[X] = thizDt.unzipAlong(path) match + case (Right(_), zpath) => newDt.rezip(zpath) + case _ => thizDt + + /** set dirTree at path creating new directories with default values if needed + */ + def insertDirAt(path: Path, dt: DirTree[X], default: X): DirTree[X] = + thizDt.unzipAlong(path) match + case (Left(path), zpath) => dt.rezip(path.map(p => pure(default) -> p).appendedAll(zpath)) + case (Right(dt), zpath) => dt.rezip(zpath) diff --git a/cache/shared/src/main/scala/run/cosy/http/cache/InterpretedCacheMiddleware.scala b/cache/shared/src/main/scala/run/cosy/http/cache/InterpretedCacheMiddleware.scala new file mode 100644 index 0000000..78ce940 --- /dev/null +++ b/cache/shared/src/main/scala/run/cosy/http/cache/InterpretedCacheMiddleware.scala @@ -0,0 +1,50 @@ +/* + * Copyright 2021 bblfish.net + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.cosy.http.cache + +import cats.data.Kleisli +import cats.effect.{Clock, Concurrent, Resource} +import io.chrisdavenport.mules.http4s.internal.Caching +import io.chrisdavenport.mules.http4s.{CacheItem, CacheType, CachedResponse} +import org.http4s.* +import org.http4s.client.Client +import cats.arrow.FunctionK + +/** Interpreted Cache don't just cache the resource but the interpretation of that resource, e.g. + * the parsed JSON or XML, or an RDF Graph or Quads + */ +object InterpretedCacheMiddleware: + type InterpClient[F[_], G[_], T] = Kleisli[G, Request[F], CachedResponse[T]] + + def client[F[_]: Concurrent: Clock, T]( + cache: TreeDirCache[F, CacheItem[T]], + interpret: Response[F] => F[CachedResponse[T]], + cacheType: CacheType = CacheType.Private + ): Client[F] => InterpClient[F, Resource[F, *], T] = (client: Client[F]) => + Kleisli( + Caching[F, T](cache, interpret, cacheType).request(Kleisli(client.run), Resource.liftK) + ) + + def app[F[_]: Concurrent: Clock, T]( + cache: TreeDirCache[F, CacheItem[T]], + interpret: Response[F] => F[CachedResponse[T]], + cacheType: CacheType = CacheType.Private + ): HttpApp[F] => InterpClient[F, F, T] = (app: HttpApp[F]) => + Kleisli( + Caching[F, T](cache, interpret, cacheType) + .request(Kleisli(app.run), cats.arrow.FunctionK.id[F]) + ) diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/CacheTest.scala b/cache/shared/src/test/scala/run/cosy/http/cache/CacheTest.scala index b9a4d6f..3ce8165 100644 --- a/cache/shared/src/test/scala/run/cosy/http/cache/CacheTest.scala +++ b/cache/shared/src/test/scala/run/cosy/http/cache/CacheTest.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 bblfish.net + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package run.cosy.http.cache import cats.MonadError @@ -6,77 +22,79 @@ import munit.CatsEffectSuite import org.http4s.Uri class CacheTest extends CatsEffectSuite: - import TreeDirCache.* - import cats.MonadError.* - import run.cosy.http.cache.DirTree + import TreeDirCache.* + import cats.MonadError.* + import run.cosy.http.cache.DirTree - def mkCache[X]: IO[TreeDirCache[IO, X]] = - for wc <- Ref.of[IO, WebCache[X]](Map.empty) - yield new TreeDirCache[IO, X](wc) + def mkCache[X]: IO[TreeDirCache[IO, X]] = + for wc <- Ref.of[IO, WebCache[X]](Map.empty) + yield new TreeDirCache[IO, X](wc) - val bbl = Uri.unsafeFromString("https://bblfish.net/people/henry/card#me") - val bblPplDir = Uri.unsafeFromString("https://bblfish.net/people/") - val bblRoot = Uri.unsafeFromString("https://bblfish.net/") - val anais = Uri.unsafeFromString("https://bblfish.net/people/anais/card#i") + val bbl = Uri.unsafeFromString("https://bblfish.net/people/henry/card#me") + val bblPplDir = Uri.unsafeFromString("https://bblfish.net/people/") + val bblRoot = Uri.unsafeFromString("https://bblfish.net/") + val anais = Uri.unsafeFromString("https://bblfish.net/people/anais/card#i") - val cacheIO: IO[TreeDirCache[IO, Int]] = mkCache[Int] + val cacheIO: IO[TreeDirCache[IO, Int]] = mkCache[Int] - test("test url paths") { - assertEquals(bbl.path.segments.map(_.toString), Vector("people", "henry", "card")) - assertEquals(bblPplDir.path.segments.map(_.toString), Vector("people")) - } + test("test url paths") { + assertEquals(bbl.path.segments.map(_.toString), Vector("people", "henry", "card")) + assertEquals(bblPplDir.path.segments.map(_.toString), Vector("people")) + } - test("first test") { - for - cache <- cacheIO - x <- cache.insert(bbl, 3) - y <- cache.lookup(bbl) - yield - assertEquals(x, ()) - assertEquals(y, Some(3)) - } + test("first test") { + for + cache <- cacheIO + x <- cache.insert(bbl, 3) + y <- cache.lookup(bbl) + yield + assertEquals(x, ()) + assertEquals(y, Some(3)) + } - test("second test, does not capture the history of changes from the first.") { - val iofail: IO[Unit] = for - cache <- cacheIO - y <- cache.lookup(bbl) - yield () - interceptIO[ServerNotFound](iofail) - } + test("second test, does not capture the history of changes from the first.") { + val iofail: IO[Unit] = + for + cache <- cacheIO + y <- cache.lookup(bbl) + yield () + interceptIO[ServerNotFound](iofail) + } - test("test searching for a parent") { - val ioCache = for - cache <- cacheIO - _ <- cache.insert(bbl, 3) - _ <- cache.insert(bblPplDir, 2) - _ <- cache.insert(bblRoot, 0) - x <- cache.lookup(bblPplDir) - y <- cache.lookup(anais) - _ <- cache.insert(anais, 12) - y2 <- cache.lookup(anais) - z <- cache.findClosest(bbl) { _ == Some(0) } - w <- cache.findClosest(anais) { _ == Some(2) } - yield - assertEquals(x, Some(2)) - assertEquals(y, None) - assertEquals(y2, Some(12)) - assertEquals(z, Some(0)) - assertEquals(w, Some(2)) - cache + test("test searching for a parent") { + val ioCache = + for + cache <- cacheIO + _ <- cache.insert(bbl, 3) + _ <- cache.insert(bblPplDir, 2) + _ <- cache.insert(bblRoot, 0) + x <- cache.lookup(bblPplDir) + y <- cache.lookup(anais) + _ <- cache.insert(anais, 12) + y2 <- cache.lookup(anais) + z <- cache.findClosest(bbl) { _ == Some(0) } + w <- cache.findClosest(anais) { _ == Some(2) } + yield + assertEquals(x, Some(2)) + assertEquals(y, None) + assertEquals(y2, Some(12)) + assertEquals(z, Some(0)) + assertEquals(w, Some(2)) + cache - for - cache <- ioCache - a <- cache.lookup(anais) - _ <- cache.delete(bblPplDir) - x <- cache.lookup(bblPplDir) - a2 <- cache.lookup(anais) - _ <- cache.deleteBelow(bblPplDir) - a3 <- cache.lookup(anais) - y3 <- cache.lookup(bbl) - yield - assertEquals(a, Some(12)) - assertEquals(x, None) - assertEquals(a2, Some(12)) - assertEquals(a3, None) - assertEquals(y3, None) - } + for + cache <- ioCache + a <- cache.lookup(anais) + _ <- cache.delete(bblPplDir) + x <- cache.lookup(bblPplDir) + a2 <- cache.lookup(anais) + _ <- cache.deleteBelow(bblPplDir) + a3 <- cache.lookup(anais) + y3 <- cache.lookup(bbl) + yield + assertEquals(a, Some(12)) + assertEquals(x, None) + assertEquals(a2, Some(12)) + assertEquals(a3, None) + assertEquals(y3, None) + } diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/DirTreeTest.scala b/cache/shared/src/test/scala/run/cosy/http/cache/DirTreeTest.scala index 63da085..3bac53b 100644 --- a/cache/shared/src/test/scala/run/cosy/http/cache/DirTreeTest.scala +++ b/cache/shared/src/test/scala/run/cosy/http/cache/DirTreeTest.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 bblfish.net + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package run.cosy.http.cache import scala.collection.immutable.HashMap @@ -5,100 +21,100 @@ import DirTree.* import org.http4s.Uri class DirTreeTest extends munit.FunSuite: - import scala.language.implicitConversions - implicit def s2p(s: String): Uri.Path.Segment = Uri.Path.Segment(s) - implicit def ps2p(ps: Seq[String]): Uri.Path = Uri.Path(ps.map(s2p).toVector) - def Pth(s: String*): Seq[Uri.Path.Segment] = s.map(s2p) - def Mp(sdt: (String, DT)*): Map[Uri.Path.Segment, DT] = - HashMap.from(sdt.map((s, dt) => Uri.Path.Segment(s) -> dt)) - type DT = DirTree[Int] - val srcTstScHello = Pth("src", "test", "scala", "hello.txt") - val fooBarBaz = Pth("foo", "bar", "baz") - val srcTst = srcTstScHello.take(2) - val src = srcTst.take(1) - val srcTsSc = srcTstScHello.take(3) - val one: DirTree[Int] = DirTree.pure[Int](1) - val twoOne = DirTree(2, Mp("test" -> one)) - val threeTwoOne = DirTree(3, Mp("src" -> twoOne)) + import scala.language.implicitConversions + implicit def s2p(s: String): Uri.Path.Segment = Uri.Path.Segment(s) + implicit def ps2p(ps: Seq[String]): Uri.Path = Uri.Path(ps.map(s2p).toVector) + def Pth(s: String*): Seq[Uri.Path.Segment] = s.map(s2p) + def Mp(sdt: (String, DT)*): Map[Uri.Path.Segment, DT] = HashMap + .from(sdt.map((s, dt) => Uri.Path.Segment(s) -> dt)) + type DT = DirTree[Int] + val srcTstScHello = Pth("src", "test", "scala", "hello.txt") + val fooBarBaz = Pth("foo", "bar", "baz") + val srcTst = srcTstScHello.take(2) + val src = srcTst.take(1) + val srcTsSc = srcTstScHello.take(3) + val one: DirTree[Int] = DirTree.pure[Int](1) + val twoOne = DirTree(2, Mp("test" -> one)) + val threeTwoOne = DirTree(3, Mp("src" -> twoOne)) - import DirTree.* - val testScHello = srcTstScHello.drop(1) + import DirTree.* + val testScHello = srcTstScHello.drop(1) - test("findClosest") { - assertEquals(one.find(Nil), (Nil, 1)) - assertEquals(one.find(fooBarBaz), (fooBarBaz, 1)) - assertEquals(one.find(srcTstScHello), (srcTstScHello, 1)) + test("findClosest") { + assertEquals(one.find(Nil), (Nil, 1)) + assertEquals(one.find(fooBarBaz), (fooBarBaz, 1)) + assertEquals(one.find(srcTstScHello), (srcTstScHello, 1)) - assertEquals(twoOne.find(Nil), (Nil, 2)) - assertEquals(twoOne.find(fooBarBaz), (fooBarBaz, 2)) - assertEquals(twoOne.find(testScHello), (testScHello.drop(1), 1)) + assertEquals(twoOne.find(Nil), (Nil, 2)) + assertEquals(twoOne.find(fooBarBaz), (fooBarBaz, 2)) + assertEquals(twoOne.find(testScHello), (testScHello.drop(1), 1)) - assertEquals(threeTwoOne.find(Nil), (Nil, 3)) - assertEquals(threeTwoOne.find(fooBarBaz), (fooBarBaz, 3)) - assertEquals(threeTwoOne.find(srcTstScHello), (srcTstScHello.drop(2), 1)) + assertEquals(threeTwoOne.find(Nil), (Nil, 3)) + assertEquals(threeTwoOne.find(fooBarBaz), (fooBarBaz, 3)) + assertEquals(threeTwoOne.find(srcTstScHello), (srcTstScHello.drop(2), 1)) - } + } - test("toClosestPath") { - assertEquals( - threeTwoOne.unzipAlong(Nil), - (Right(threeTwoOne), Nil) - ) - assertEquals( - threeTwoOne.unzipAlong(Pth("src")), - (Right(twoOne), Seq(ZLink(threeTwoOne, "src"))) - ) - assertEquals( - threeTwoOne.unzipAlong(Pth("src", "test")), - (Right(one), Seq(ZLink(twoOne, "test"), ZLink(threeTwoOne, "src"))) - ) - // note that "scala" here is not in One. It is pointing to a place we may want something to be. - assertEquals( - threeTwoOne.unzipAlong(Pth("src", "test", "scala")), - (Left(Nil), Seq(one -> "scala", twoOne -> "test", threeTwoOne -> "src")) - ) - assertEquals( - threeTwoOne.unzipAlong(Pth("src", "test", "scala", "hello.txt")), - (Left(Pth("hello.txt")), List(one -> "scala", twoOne -> "test", threeTwoOne -> "src")) - ) - } + test("toClosestPath") { + assertEquals( + threeTwoOne.unzipAlong(Nil), + (Right(threeTwoOne), Nil) + ) + assertEquals( + threeTwoOne.unzipAlong(Pth("src")), + (Right(twoOne), Seq(ZLink(threeTwoOne, "src"))) + ) + assertEquals( + threeTwoOne.unzipAlong(Pth("src", "test")), + (Right(one), Seq(ZLink(twoOne, "test"), ZLink(threeTwoOne, "src"))) + ) + // note that "scala" here is not in One. It is pointing to a place we may want something to be. + assertEquals( + threeTwoOne.unzipAlong(Pth("src", "test", "scala")), + (Left(Nil), Seq(one -> "scala", twoOne -> "test", threeTwoOne -> "src")) + ) + assertEquals( + threeTwoOne.unzipAlong(Pth("src", "test", "scala", "hello.txt")), + (Left(Pth("hello.txt")), List(one -> "scala", twoOne -> "test", threeTwoOne -> "src")) + ) + } - test("rezip") { - assertEquals(one.rezip(Nil), one) - assertEquals(one.rezip(Seq(DirTree.pure(2) -> "test")), twoOne) - assertEquals(one.rezip(Seq(DirTree.pure(2) -> "test", DirTree.pure(3) -> "src")), threeTwoOne) - } + test("rezip") { + assertEquals(one.rezip(Nil), one) + assertEquals(one.rezip(Seq(DirTree.pure(2) -> "test")), twoOne) + assertEquals(one.rezip(Seq(DirTree.pure(2) -> "test", DirTree.pure(3) -> "src")), threeTwoOne) + } - test("setAt") { - assertEquals(one.insertAt(Nil, 2, 0), DirTree.pure(2)) - assertEquals(one.insertAt(Nil, -1, 0), DirTree.pure(-1)) - assertEquals(one.insertAt(Seq("two"), 2, 0), DirTree(1, Mp("two" -> DirTree.pure(2)))) - assertEquals( - one.insertAt(Pth("two", "three"), 2, 0), - DirTree(1, Mp("two" -> DirTree(0, Mp("three" -> DirTree.pure(2))))) - ) - val tz1 = DirTree(3, Mp("zero" -> DirTree(0, Mp("one" -> DirTree.pure(1))))) - assertEquals(DirTree.pure(3).insertAt(Seq("zero", "one"), 1, 0), tz1) - val zom1t = DirTree( - 3, - Mp( - "zero" -> DirTree( - 0, - Mp("one" -> DirTree(1, Mp("minus1" -> DirTree(-1, Mp("two" -> DirTree.pure(2)))))) - ) - ) - ) - assertEquals(tz1.insertAt(Seq("zero", "one", "minus1", "two"), 2, -1), zom1t) - assertEquals( - zom1t.insertAt(Pth("zero", "one"), 55555, -2), - DirTree( - 3, - Mp( - "zero" -> DirTree( - 0, - Mp("one" -> DirTree(55555, Mp("minus1" -> DirTree(-1, Mp("two" -> DirTree.pure(2)))))) - ) - ) - ) - ) - } + test("setAt") { + assertEquals(one.insertAt(Nil, 2, 0), DirTree.pure(2)) + assertEquals(one.insertAt(Nil, -1, 0), DirTree.pure(-1)) + assertEquals(one.insertAt(Seq("two"), 2, 0), DirTree(1, Mp("two" -> DirTree.pure(2)))) + assertEquals( + one.insertAt(Pth("two", "three"), 2, 0), + DirTree(1, Mp("two" -> DirTree(0, Mp("three" -> DirTree.pure(2))))) + ) + val tz1 = DirTree(3, Mp("zero" -> DirTree(0, Mp("one" -> DirTree.pure(1))))) + assertEquals(DirTree.pure(3).insertAt(Seq("zero", "one"), 1, 0), tz1) + val zom1t = DirTree( + 3, + Mp( + "zero" -> DirTree( + 0, + Mp("one" -> DirTree(1, Mp("minus1" -> DirTree(-1, Mp("two" -> DirTree.pure(2)))))) + ) + ) + ) + assertEquals(tz1.insertAt(Seq("zero", "one", "minus1", "two"), 2, -1), zom1t) + assertEquals( + zom1t.insertAt(Pth("zero", "one"), 55555, -2), + DirTree( + 3, + Mp( + "zero" -> DirTree( + 0, + Mp("one" -> DirTree(55555, Mp("minus1" -> DirTree(-1, Mp("two" -> DirTree.pure(2)))))) + ) + ) + ) + ) + } diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala b/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala new file mode 100644 index 0000000..81bf653 --- /dev/null +++ b/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala @@ -0,0 +1,117 @@ +/* + * Copyright 2021 bblfish.net + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.cosy.http.cache + +import cats.effect.{IO, Async} +import cats.syntax.all.* +import org.http4s.* +import org.http4s.headers.* +import org.http4s.implicits.* +import org.http4s.Uri.Path.Root +import org.http4s.Method.{GET, HEAD} +import scodec.bits.ByteVector +import org.http4s.dsl.io.* +import run.cosy.http.cache.TreeDirCache.WebCache +import cats.effect.kernel.Ref +import cats.data.Kleisli +import io.chrisdavenport.mules.http4s.CachedResponse +import io.chrisdavenport.mules.http4s.CacheItem + +object Test: +// Return true if match succeeds; otherwise false +// def check[A](actual: IO[Response[IO]], expectedStatus: Status, expectedBody: Option[A])(using +// ev: EntityDecoder[IO, A] +// ): Boolean = +// val actualResp = actual.unsafeRunSync() +// val statusCheck = actualResp.status == expectedStatus +// val bodyCheck = expectedBody.fold[Boolean]( +// // Verify Response's body is empty. +// actualResp.body.compile.toVector.unsafeRunSync().isEmpty +// )(expected => actualResp.as[A].unsafeRunSync() == expected) +// statusCheck && bodyCheck +end Test + +class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: + test("test web") { + Web.httpRoutes[IO].orNotFound.run(Request[IO](uri = Uri(path = Root))).map { response => + assertEquals(response.status, Status.Ok) + assertEquals( + response.headers, + Web.headers("/") + ) + } >> Web.httpRoutes[IO].orNotFound.run( + Request[IO](uri = + Uri(path = Uri.Path.unsafeFromString("/people/henry/blog/2023/04/01/world-at-peace")) + ) + ).map { response => + assertEquals(response.status, Status.Ok) + assertEquals( + response.headers, + Web.headers("world-at-peace", Some("/people/henry/blog/"), MediaType.text.plain) + ) + assertEquals( + bytesToString(response.body.compile.toVector.unsafeRunSync()), + "Hello World!" + ) + } + } + def bytesToString(bytes: Vector[Byte]): String = bytes.map(_.toChar).mkString + + test("test string cache") { + for ref <- Ref.of[IO, WebCache[CacheItem[String]]](Map.empty) + yield + val clientMiddleWare = InterpretedCacheMiddleware.app[IO, String]( + TreeDirCache[IO, CacheItem[String]](ref), + (response: Response[IO]) => + response.bodyText.compile.toVector.map { vec => + CachedResponse( + response.status, + response.httpVersion, + response.headers, + Some(vec.mkString) + ) + } + ) + + val bbl: Uri = Uri + .unsafeFromString("https://bblfish.net/people/henry/blog/2023/04/01/world-at-peace") + assertNotEquals(bbl, null) + val interpretedClient = clientMiddleWare(Web.httpRoutes[IO].orNotFound) + val req = Request[IO](Method.GET, bbl) + val resp: CachedResponse[String] = interpretedClient.run(req).unsafeRunSync() + assertEquals(resp.status, Status.Ok) + assertEquals( + resp.headers, + Web.headers("world-at-peace", Some("/people/henry/blog/"), MediaType.text.plain) + ) + assertEquals( + resp.body, + Some("Hello World!") + ) + // val resp2 = client(Web.httpRoutes[IO].orNotFound).run(req).unsafeRunSync() + // assertEquals(resp2.status, Status.Ok) + // assertEquals( + // resp2.headers, + // Web.headers("/") + // ) + // assertEquals( + // resp2.body.compile.toVector.unsafeRunSync(), + // ByteVector("Hello World!".getBytes()) + // ) + } + +end InterpretedCacheMiddleTest diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala b/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala new file mode 100644 index 0000000..ea2a0a3 --- /dev/null +++ b/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala @@ -0,0 +1,149 @@ +/* + * Copyright 2021 bblfish.net + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.cosy.http.cache + +import cats.effect.{IO, Async} +import cats.syntax.all.* +import org.http4s.* +import org.http4s.headers.* +import org.http4s.implicits.* +import org.http4s.Uri.Path.Root +import org.http4s.Method.{GET, HEAD} +import scodec.bits.ByteVector +import org.http4s.dsl.io.* +// import io.chrisdavenport.mules.http4s.CachedResponse.body + +object Web: + // implicit def toEnt[F](str: String): Entity[F] = + + extension (uri: Uri) + /** Uri("/") + ".acl" == Uri("/.acl") and Uri("foo")+".acl" == Uri("foo.acl") */ + def +(ext: String): Uri = + if uri.path.endsWithSlash then uri.copy(path = uri.path / ext) + else + uri.copy(path = + Uri.Path(uri.path.segments.dropRight(1)) / (uri.path.segments.last.toString + ext) + ) + + def headers( + doc: String, + default: Option[String] = None, + mime: MediaType = MediaType("text", "turtle") + ): Headers = + val docAcrUri = Uri.unsafeFromString(doc + ".acr") + val defaultUri: Option[Uri] = default.map(Uri.unsafeFromString) + Headers( + `Content-Type`(mime), + Link( + LinkValue(docAcrUri, rel = Some("acl")), + defaultUri.map(ac => LinkValue(ac, rel = Some("defaultAccessContainer"))).toSeq* + ), + Allow(GET, HEAD) + ) + + val rootDir = Uri(path = Root) + // relative URLs + val thisDoc = Uri.unsafeFromString("") + val thisDir = Uri.unsafeFromString(".") + + def httpRoutes[F[_]](using + AS: Async[F] + ): HttpRoutes[F] = HttpRoutes.of[F] { + case GET -> Root => AS.pure( + Response[F]( + status = Status.Ok, + entity = Entity.strict(ByteVector(""" + |@prefix ldp: . + | + |<> a ldp:BasicContainer; + | ldp:contains . + |""".stripMargin.getBytes())), + headers = headers("/") + ) + ) + + case GET -> Root / ".acr" => AS.pure( + Response[F]( + status = Status.Ok, + entity = Entity.strict(ByteVector(""" + |@prefix wac: . + |@prefix foaf: . + | + |<#R1> a wac:Authorization; + | wac:mode wac:Control; + | wac:agent ; + | wac:default <.> . + | + |<#R2> a wac:Authorization; + | wac:mode wac:Read; + | wac:agentClass foaf:Agent; + | wac:accessTo <.> ; + | wac:default <.> . + |""".stripMargin.getBytes())), + headers = headers("/") + ) + ) + + case GET -> Root / "people" / "henry" / "card" => AS.pure( + Response( + status = Status.Ok, + entity = Entity.strict(ByteVector(""" + |<#i> a foaf:Person; + | foaf.name "Henry Story"; + | foaf.knows . + """.stripMargin.getBytes())), + headers = headers("card", Some("/")) + ) + ) + + case GET -> Root / "people" / "henry" / "blog" / "2023" / "04" / "01" / "world-at-peace" => + OK[F]( + "Hello World!", + headers("world-at-peace", Some("/people/henry/blog/"), MediaType.text.plain) + ) + + case GET -> Root / "people" / "henry" / "blog" / ".acr" => OK[F]( + """ + |@prefix wac: . + |@prefix foaf: . + | + |<#R1> a wac:Authorization; + | wac:mode wac:Control; + | wac:agent ; + | wac:default <.> . + | + |<#R2> a wac:Authorization; + | wac:mode wac:Read; + | wac:agentClass foaf:Agent; + | wac:default <.> . + | + |<#R3> a wac:Authorization; + | wac:mode wac:Read; + | wac:agentClass foaf:Agent; + | wac:default <.> . + """.stripMargin, + headers("") + ) + } + + def OK[F[_]: Async](body: String, headers: Headers): F[Response[F]] = Async[F].pure( + Response[F]( + status = Status.Ok, + entity = Entity.strict(ByteVector(body.getBytes())), + headers = headers + ) + ) diff --git a/ioExt4s/shared/src/main/scala/run/cosy/ld/http4s/RDFDecoders.scala b/ioExt4s/shared/src/main/scala/run/cosy/ld/http4s/RDFDecoders.scala index 144bfde..208a1de 100644 --- a/ioExt4s/shared/src/main/scala/run/cosy/ld/http4s/RDFDecoders.scala +++ b/ioExt4s/shared/src/main/scala/run/cosy/ld/http4s/RDFDecoders.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -48,59 +48,57 @@ class RDFDecoders[F[_], Rdf <: RDF](using jsonLDReader: RelRDFReader[Rdf, Try, JsonLd] ): - private def decoderForRdfReader[T](mt: MediaRange, mts: MediaRange*)( - reader: RelRDFReader[Rdf, Try, T], - errmsg: String - ): EntityDecoder[F, rGraph[Rdf]] = - given defaultCharset: Charset = Charset.`UTF-8` - EntityDecoder.decodeBy[F, rGraph[Rdf]](mt, mts: _*) { msg => - val txt: F[String] = EntityDecoder.decodeText[F](msg) - EitherT[F, DecodeFailure, rGraph[Rdf]]( - cc.flatMap[String, Either[DecodeFailure, rGraph[Rdf]]](txt) { s => - reader.read(new java.io.StringReader(s)) match - case Success(rg) => cc.pure(Right(rg)) - case Failure(err) => - cc.pure(Left(MalformedMessageBodyFailure(errmsg, Some(err)))) - } - ) - } - import MediaType.{application, text} - import MediaType.application.`ld+json` + private def decoderForRdfReader[T](mt: MediaRange, mts: MediaRange*)( + reader: RelRDFReader[Rdf, Try, T], + errmsg: String + ): EntityDecoder[F, rGraph[Rdf]] = + given defaultCharset: Charset = Charset.`UTF-8` + EntityDecoder.decodeBy[F, rGraph[Rdf]](mt, mts*) { msg => + val txt: F[String] = EntityDecoder.decodeText[F](msg) + EitherT[F, DecodeFailure, rGraph[Rdf]]( + cc.flatMap[String, Either[DecodeFailure, rGraph[Rdf]]](txt) { s => + reader.read(new java.io.StringReader(s)) match + case Success(rg) => cc.pure(Right(rg)) + case Failure(err) => cc.pure(Left(MalformedMessageBodyFailure(errmsg, Some(err)))) + } + ) + } + import MediaType.{application, text} + import MediaType.application.`ld+json` - // add to offiwcial types along with those described here: - // https://github.com/co-operating-systems/Reactive-SoLiD/blob/master/src/main/scala/run/cosy/http/RDFMediaTypes.scala - lazy val ntriples: MediaType = - new MediaType( - "application", - "n-triples", - Compressible, - NotBinary, - List("nt") - ) + // add to offiwcial types along with those described here: + // https://github.com/co-operating-systems/Reactive-SoLiD/blob/master/src/main/scala/run/cosy/http/RDFMediaTypes.scala + lazy val ntriples: MediaType = new MediaType( + "application", + "n-triples", + Compressible, + NotBinary, + List("nt") + ) - val turtleDecoder: EntityDecoder[F, rGraph[Rdf]] = decoderForRdfReader(text.turtle, ntriples)( - turtleReader, - "Rdf Turtle Reader failed" - ) - val rdfxmlDecoder: EntityDecoder[F, rGraph[Rdf]] = decoderForRdfReader(application.`rdf+xml`)( - rdfXmlReader, - "Rdf Rdf/XML Reader failed" - ) + val turtleDecoder: EntityDecoder[F, rGraph[Rdf]] = decoderForRdfReader(text.turtle, ntriples)( + turtleReader, + "Rdf Turtle Reader failed" + ) + val rdfxmlDecoder: EntityDecoder[F, rGraph[Rdf]] = decoderForRdfReader(application.`rdf+xml`)( + rdfXmlReader, + "Rdf Rdf/XML Reader failed" + ) // val ntriplesDecoder = decoderForRdfReader(application.`n-triples`)( // ntriplesReader, "NTriples Reader failed" // ) - val jsonldDecoder: EntityDecoder[F, rGraph[Rdf]] = decoderForRdfReader(application.`ld+json`)( - jsonLDReader, - "Json-LD Reader failed" - ) + val jsonldDecoder: EntityDecoder[F, rGraph[Rdf]] = decoderForRdfReader(application.`ld+json`)( + jsonLDReader, + "Json-LD Reader failed" + ) - given allrdf: EntityDecoder[F, rGraph[Rdf]] = - turtleDecoder orElse rdfxmlDecoder orElse jsonldDecoder + given allrdf: EntityDecoder[F, rGraph[Rdf]] = + turtleDecoder orElse rdfxmlDecoder orElse jsonldDecoder - val allRdfAccept = Accept( - text.turtle.withQValue(QValue.One), - ntriples.withQValue(QValue.One), - application.`ld+json`.withQValue(qValue"0.8"), - application.`rdf+xml`.withQValue(qValue"0.8") - ) + val allRdfAccept = Accept( + text.turtle.withQValue(QValue.One), + ntriples.withQValue(QValue.One), + application.`ld+json`.withQValue(qValue"0.8"), + application.`rdf+xml`.withQValue(qValue"0.8") + ) end RDFDecoders diff --git a/ioExt4s/shared/src/main/scala/run/cosy/web/util/UrlUtil.scala b/ioExt4s/shared/src/main/scala/run/cosy/web/util/UrlUtil.scala index caeb438..f7b5bb3 100644 --- a/ioExt4s/shared/src/main/scala/run/cosy/web/util/UrlUtil.scala +++ b/ioExt4s/shared/src/main/scala/run/cosy/web/util/UrlUtil.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,47 +23,50 @@ import org.w3.banana.{Ops, RDF} import scala.util.Try -object UrlUtil { +object UrlUtil: - extension (h4uri: org.http4s.Uri) def toLL: ll.Url = http4sUrlToLLUrl(h4uri) + extension (h4uri: org.http4s.Uri) def toLL: ll.Url = http4sUrlToLLUrl(h4uri) - extension (llUri: ll.Url) def toh4: org.http4s.Uri = llUrltoHttp4s(llUri) + extension (llUri: ll.Url) def toh4: org.http4s.Uri = llUrltoHttp4s(llUri) - extension [R <: RDF](uri: RDF.URI[R])(using ops: Ops[R]) - def toLL: Try[ll.Uri] = - import ops.{*, given} - ll.Uri.parseTry(uri.value) + extension [R <: RDF](uri: RDF.URI[R])(using ops: Ops[R]) + def toLL: Try[ll.Uri] = + import ops.{*, given} + ll.Uri.parseTry(uri.value) - // ignoring username:password urls - def http4sUrlToLLUrl(u: org.http4s.Uri): ll.Url = - import u.{host as h4host, path as h4path, query as h4query, port as h4port, scheme as h4scheme} - ll.Url( - h4scheme.map(_.value).getOrElse(null), - null, - null, - h4host.map(_.value).getOrElse(null), - h4port.getOrElse(-1), - h4path.toString, - ll.QueryString.parse(h4query.toString), - null - ) - end http4sUrlToLLUrl + // ignoring username:password urls + def http4sUrlToLLUrl(u: org.http4s.Uri): ll.Url = + import u.{ + host as h4host, + path as h4path, + query as h4query, + port as h4port, + scheme as h4scheme + } + ll.Url( + h4scheme.map(_.value).getOrElse(null), + null, + null, + h4host.map(_.value).getOrElse(null), + h4port.getOrElse(-1), + h4path.toString, + ll.QueryString.parse(h4query.toString), + null + ) + end http4sUrlToLLUrl - def llUrltoHttp4s(u: ll.Url): org.http4s.Uri = - val h4schm: Option[h4Uri.Scheme] = u.schemeOption.flatMap { sch => - h4Uri.Scheme.fromString(sch).toOption - } - val h4Auth: Option[h4Uri.Authority] = u.authorityOption.flatMap { llAuth => - val ui: Option[h4Uri.UserInfo] = - llAuth.userInfo.map(ui => h4Uri.UserInfo(ui.user, ui.password)) - val hostOpt: Option[h4Uri.Host] = ip4s.Host - .fromString(llAuth.host.value) - .map(h4Uri.Host.fromIp4sHost) - hostOpt.map(host => h4Uri.Authority(ui, host, llAuth.port)) - } - val h4path: h4Uri.Path = h4Uri.Path(u.path.parts.map(str => h4Uri.Path.Segment(str))) - val h4q: org.http4s.Query = org.http4s.Query(u.query.params*) - org.http4s.Uri(h4schm, h4Auth, h4path, h4q, u.fragment) - end llUrltoHttp4s - -} + def llUrltoHttp4s(u: ll.Url): org.http4s.Uri = + val h4schm: Option[h4Uri.Scheme] = u.schemeOption.flatMap { sch => + h4Uri.Scheme.fromString(sch).toOption + } + val h4Auth: Option[h4Uri.Authority] = u.authorityOption.flatMap { llAuth => + val ui: Option[h4Uri.UserInfo] = llAuth.userInfo + .map(ui => h4Uri.UserInfo(ui.user, ui.password)) + val hostOpt: Option[h4Uri.Host] = ip4s.Host.fromString(llAuth.host.value) + .map(h4Uri.Host.fromIp4sHost) + hostOpt.map(host => h4Uri.Authority(ui, host, llAuth.port)) + } + val h4path: h4Uri.Path = h4Uri.Path(u.path.parts.map(str => h4Uri.Path.Segment(str))) + val h4q: org.http4s.Query = org.http4s.Query(u.query.params*) + org.http4s.Uri(h4schm, h4Auth, h4path, h4q, u.fragment) + end llUrltoHttp4s diff --git a/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala b/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala index 7080a6e..763cc26 100644 --- a/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala +++ b/ldes/shared/src/main/scala/run/cosy/ld/PNGraph.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,11 +23,11 @@ import org.w3.banana.RDF.{Graph, Node, Statement, URI} import org.w3.banana.{Ops, RDF} trait Web[F[_]: Concurrent, R <: RDF]: - /** get a Graph for the given URL (without a fragment) */ - def get(url: RDF.URI[R]): F[RDF.Graph[R]] + /** get a Graph for the given URL (without a fragment) */ + def get(url: RDF.URI[R]): F[RDF.Graph[R]] - /** get Pointed Named Graph for given url */ - def getPNG(url: RDF.URI[R]): F[UriNGraph[R]] + /** get Pointed Named Graph for given url */ + def getPNG(url: RDF.URI[R]): F[UriNGraph[R]] /** A Pointed Named Graph, ie, a pointer into a NamedGraph We don't use a case class here as * equality between PNGgraphs is complicated by the need to prove isomorphism between graphs, and @@ -38,67 +38,61 @@ trait Web[F[_]: Concurrent, R <: RDF]: * sense. png.jump makes sense only if the node is a URL */ trait PNGraph[R <: RDF]: - type Point <: RDF.URI[R] | RDF.Literal[R] | RDF.BNode[R] - def point: Point - def name: URI[R] - def graph: Graph[R] - - /** collect from the given point relations forward and backward (where possible) and return a - * graph - */ - def collect( - forward: RDF.URI[R]* - )( - backward: RDF.URI[R]* - )(using ops: Ops[R]): RDF.Graph[R] = - import ops.{*, given} - val f: Seq[Seq[RDF.Triple[R]]] = - if point.isURI || point.isBNode then - for frel <- forward - yield graph.find(point, frel, `*`).toSeq - else Seq.empty - val b: Seq[Seq[RDF.Triple[R]]] = - for brel <- backward - yield graph.find(`*`, brel, point).toSeq - Graph(f.flatten ++ b.flatten) + type Point <: RDF.URI[R] | RDF.Literal[R] | RDF.BNode[R] + def point: Point + def name: URI[R] + def graph: Graph[R] + + /** collect from the given point relations forward and backward (where possible) and return a + * graph + */ + def collect( + forward: RDF.URI[R]* + )( + backward: RDF.URI[R]* + )(using ops: Ops[R]): RDF.Graph[R] = + import ops.{*, given} + val f: Seq[Seq[RDF.Triple[R]]] = + if point.isURI || point.isBNode then + for frel <- forward + yield graph.find(point, frel, `*`).toSeq + else Seq.empty + val b: Seq[Seq[RDF.Triple[R]]] = + for brel <- backward + yield graph.find(`*`, brel, point).toSeq + Graph(f.flatten ++ b.flatten) trait SubjPNGraph[R <: RDF] extends PNGraph[R]: - type Point <: RDF.Statement.Subject[R] + type Point <: RDF.Statement.Subject[R] - def /(rel: RDF.URI[R])(using ops: Ops[R]): Seq[PNGraph[R]] = - import ops.{*, given} - graph - .find(point, rel, `*`) - .map { (tr: RDF.Triple[R]) => + def /(rel: RDF.URI[R])(using ops: Ops[R]): Seq[PNGraph[R]] = + import ops.{*, given} + graph.find(point, rel, `*`).map { (tr: RDF.Triple[R]) => tr.obj.asNode.fold( uri => new UriNGraph(uri, name, graph), bn => new BNodeNGraph[R](bn, name, graph), lit => new LiteralNGraph[R](lit, name, graph) ) - } - .toSeq - end / - - inline def rel(rel: RDF.URI[R])(using ops: Ops[R]): Seq[PNGraph[R]] = /(rel) - - def \(rel: RDF.URI[R])(using ops: Ops[R]): Seq[PNGraph[R]] = - import ops.{*, given} - graph - .find(`*`, rel, point) - .map { (tr: RDF.Triple[R]) => - val s: Statement.Subject[R] = tr.subj - s.foldSubj[PNGraph[R]]( - uri => new UriNGraph(uri, name, graph), - bn => new BNodeNGraph[R](bn, name, graph) - ) - } - .toSeq - end \ - - def hasType(tp: RDF.URI[R])(using ops: Ops[R]): Boolean = - import ops.{*, given} - val it: Iterator[RDF.Triple[R]] = graph.find(point, rdf.typ, tp) - it.hasNext + }.toSeq + end / + + inline def rel(rel: RDF.URI[R])(using ops: Ops[R]): Seq[PNGraph[R]] = /(rel) + + def \(rel: RDF.URI[R])(using ops: Ops[R]): Seq[PNGraph[R]] = + import ops.{*, given} + graph.find(`*`, rel, point).map { (tr: RDF.Triple[R]) => + val s: Statement.Subject[R] = tr.subj + s.foldSubj[PNGraph[R]]( + uri => new UriNGraph(uri, name, graph), + bn => new BNodeNGraph[R](bn, name, graph) + ) + }.toSeq + end \ + + def hasType(tp: RDF.URI[R])(using ops: Ops[R]): Boolean = + import ops.{*, given} + val it: Iterator[RDF.Triple[R]] = graph.find(point, rdf.typ, tp) + it.hasNext end SubjPNGraph @@ -109,18 +103,18 @@ case class UriNGraph[R <: RDF]( val graph: Graph[R] )(using ops: Ops[R]) extends SubjPNGraph[R]: - import ops.{*, given} - type Point = RDF.URI[R] + import ops.{*, given} + type Point = RDF.URI[R] - lazy val pointLoc: RDF.URI[R] = point.fragmentLess - lazy val isLocal: Boolean = pointLoc == name + lazy val pointLoc: RDF.URI[R] = point.fragmentLess + lazy val isLocal: Boolean = pointLoc == name //todo: things are more complex when one takes redirects into account... - def jump[F[_]: Concurrent](using web: Web[F, R]): F[UriNGraph[R]] = - import cats.syntax.functor.* - import ops.{*, given} - if isLocal then summon[Concurrent[F]].pure(this) - else web.get(pointLoc).map(g => new UriNGraph(point, pointLoc, g)) + def jump[F[_]: Concurrent](using web: Web[F, R]): F[UriNGraph[R]] = + import cats.syntax.functor.* + import ops.{*, given} + if isLocal then summon[Concurrent[F]].pure(this) + else web.get(pointLoc).map(g => new UriNGraph(point, pointLoc, g)) end UriNGraph @@ -130,7 +124,7 @@ class BNodeNGraph[R <: RDF]( val name: URI[R], val graph: Graph[R] ) extends SubjPNGraph[R]: - type Point = RDF.BNode[R] + type Point = RDF.BNode[R] /** A PNG where the point is a BNode */ class LiteralNGraph[R <: RDF]( @@ -138,79 +132,69 @@ class LiteralNGraph[R <: RDF]( val name: URI[R], val graph: Graph[R] ) extends PNGraph[R]: - type Point = RDF.Literal[R] - - // todo: this is duplicated code with SubjPNGraph. - def \(rel: RDF.URI[R])(using ops: Ops[R]): Seq[PNGraph[R]] = - import ops.{*, given} - graph - .find(`*`, rel, point) - .map { (tr: RDF.Triple[R]) => + type Point = RDF.Literal[R] + + // todo: this is duplicated code with SubjPNGraph. + def \(rel: RDF.URI[R])(using ops: Ops[R]): Seq[PNGraph[R]] = + import ops.{*, given} + graph.find(`*`, rel, point).map { (tr: RDF.Triple[R]) => tr.subj.foldSubj( uri => UriNGraph(uri, name, graph), bn => BNodeNGraph(bn, name, graph) ) - } - .toSeq + }.toSeq end LiteralNGraph object PNGraph: - extension [F[_]: cats.effect.Concurrent, R <: RDF]( - pngs: Seq[PNGraph[R]] - )(using web: Web[F, R], ops: Ops[R]) - // jump on nodes in the sequence that can be jumped - def jump: fs2.Stream[F, PNGraph[R]] = - import ops.given - var local: List[PNGraph[R]] = Nil - var remoteGs: List[UriNGraph[R]] = Nil - pngs.foreach { - case ug: UriNGraph[R] if !ug.isLocal => remoteGs = ug :: remoteGs - case other => local = other :: local - } - import cats.syntax.all.given - - val in: fs2.Stream[Pure, PNGraph[R]] = fs2.Stream.chunk(fs2.Chunk.seq(local)) - val out: fs2.Stream[F, PNGraph[R]] = - fs2.Stream - .emits(remoteGs) - // todo: group the urls by domain (groupAdjacentBy) and run each with two threads max - .parEvalMapUnordered(5) { (remote: UriNGraph[R]) => - // todo: we should deal with errors fetching graphs - // web should perhaps return a UriNGraph - web.get(remote.pointLoc).map(g => UriNGraph[R](remote.point, remote.pointLoc, g)) - } - in ++ out - end jump - - def rel(relation: RDF.URI[R]): Seq[PNGraph[R]] = - pngs.collect { case sub: SubjPNGraph[R] => + extension [F[_]: cats.effect.Concurrent, R <: RDF]( + pngs: Seq[PNGraph[R]] + )(using web: Web[F, R], ops: Ops[R]) + // jump on nodes in the sequence that can be jumped + def jump: fs2.Stream[F, PNGraph[R]] = + import ops.given + var local: List[PNGraph[R]] = Nil + var remoteGs: List[UriNGraph[R]] = Nil + pngs.foreach { + case ug: UriNGraph[R] if !ug.isLocal => remoteGs = ug :: remoteGs + case other => local = other :: local + } + import cats.syntax.all.given + + val in: fs2.Stream[Pure, PNGraph[R]] = fs2.Stream.chunk(fs2.Chunk.seq(local)) + val out: fs2.Stream[F, PNGraph[R]] = fs2.Stream.emits(remoteGs) + // todo: group the urls by domain (groupAdjacentBy) and run each with two threads max + .parEvalMapUnordered(5) { (remote: UriNGraph[R]) => + // todo: we should deal with errors fetching graphs + // web should perhaps return a UriNGraph + web.get(remote.pointLoc).map(g => UriNGraph[R](remote.point, remote.pointLoc, g)) + } + in ++ out + end jump + + def rel(relation: RDF.URI[R]): Seq[PNGraph[R]] = pngs.collect { case sub: SubjPNGraph[R] => sub / relation }.flatten - def filterType(tp: RDF.URI[R]): Seq[SubjPNGraph[R]] = - pngs.collect { + def filterType(tp: RDF.URI[R]): Seq[SubjPNGraph[R]] = pngs.collect { case sub: SubjPNGraph[R] if sub.hasType(tp) => sub } - extension [F[_]: cats.effect.Concurrent, R <: RDF]( - pngs: fs2.Stream[F, PNGraph[R]] - )(using ops: Ops[R]) + extension [F[_]: cats.effect.Concurrent, R <: RDF]( + pngs: fs2.Stream[F, PNGraph[R]] + )(using ops: Ops[R]) - def /(rel: RDF.URI[R]): fs2.Stream[F, PNGraph[R]] = - pngs.collect { case sub: SubjPNGraph[R] => + def /(rel: RDF.URI[R]): fs2.Stream[F, PNGraph[R]] = pngs.collect { case sub: SubjPNGraph[R] => fs2.Stream(sub / rel*) }.flatten - inline def rel(rel: RDF.URI[R]): fs2.Stream[F, PNGraph[R]] = /(rel) + inline def rel(rel: RDF.URI[R]): fs2.Stream[F, PNGraph[R]] = /(rel) - def jump(using Web[F, R]): fs2.Stream[F, SubjPNGraph[R]] = - pngs.collect { + def jump(using Web[F, R]): fs2.Stream[F, SubjPNGraph[R]] = pngs.collect { case bn: BNodeNGraph[R] => fs2.Stream(bn) - case un: UriNGraph[R] => fs2.Stream.eval(un.jump) + case un: UriNGraph[R] => fs2.Stream.eval(un.jump) }.flatten - def filterType(tp: RDF.URI[R]): fs2.Stream[F, SubjPNGraph[R]] = - pngs.collect { + def filterType(tp: RDF.URI[R]): fs2.Stream[F, SubjPNGraph[R]] = pngs.collect { case sub: SubjPNGraph[R] if sub.hasType(tp) => sub } diff --git a/ldes/shared/src/main/scala/run/cosy/ld/http4s/H4Web.scala b/ldes/shared/src/main/scala/run/cosy/ld/http4s/H4Web.scala index 3f20505..b915225 100644 --- a/ldes/shared/src/main/scala/run/cosy/ld/http4s/H4Web.scala +++ b/ldes/shared/src/main/scala/run/cosy/ld/http4s/H4Web.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,21 +35,21 @@ class H4Web[F[_]: Concurrent, R <: RDF]( )(using val rdfDecoders: RDFDecoders[F, R] ) extends Web[F, R]: - import cats.syntax.all.* - import rdfDecoders.ops - import rdfDecoders.allrdf - import ops.{*, given} + import cats.syntax.all.* + import rdfDecoders.ops + import rdfDecoders.allrdf + import ops.{*, given} - override def get(url: RDF.URI[R]): F[RDF.Graph[R]] = - for - u <- Concurrent[F].fromEither(h4s.Uri.fromString(url.value)) - doc = u.withoutFragment - rG <- client.fetchAs[RDF.rGraph[R]]( - h4s.Request(uri = doc) - ) - yield rG.resolveAgainst(http4sUrlToLLUrl(doc).toAbsoluteUrl) + override def get(url: RDF.URI[R]): F[RDF.Graph[R]] = + for + u <- Concurrent[F].fromEither(h4s.Uri.fromString(url.value)) + doc = u.withoutFragment + rG <- client.fetchAs[RDF.rGraph[R]]( + h4s.Request(uri = doc) + ) + yield rG.resolveAgainst(http4sUrlToLLUrl(doc).toAbsoluteUrl) - // todo: this should really use client.fetchPNG - override def getPNG(url: RDF.URI[R]): F[UriNGraph[R]] = - val doc = url.fragmentLess - get(doc).map(g => UriNGraph(url, doc, g)) + // todo: this should really use client.fetchPNG + override def getPNG(url: RDF.URI[R]): F[UriNGraph[R]] = + val doc = url.fragmentLess + get(doc).map(g => UriNGraph(url, doc, g)) diff --git a/ldes/shared/src/main/scala/run/cosy/ldes/LdesSpider.scala b/ldes/shared/src/main/scala/run/cosy/ldes/LdesSpider.scala index 82c8fdd..035bb2c 100644 --- a/ldes/shared/src/main/scala/run/cosy/ldes/LdesSpider.scala +++ b/ldes/shared/src/main/scala/run/cosy/ldes/LdesSpider.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,62 +29,60 @@ class LdesSpider[F[_]: Concurrent, R <: RDF](using ops: Ops[R] ): - import ops.{*, given} - import org.w3.banana.prefix - import run.cosy.ldes.prefix as ldesPre + import ops.{*, given} + import org.w3.banana.prefix + import run.cosy.ldes.prefix as ldesPre - val foaf = prefix.FOAF[R] - val tree = ldesPre.TREE[R] - val sosa = ldesPre.SOSA[R] - val wgs84 = ldesPre.WGS84[R] - val ldes = ldesPre.LDES[R] + val foaf = prefix.FOAF[R] + val tree = ldesPre.TREE[R] + val sosa = ldesPre.SOSA[R] + val wgs84 = ldesPre.WGS84[R] + val ldes = ldesPre.LDES[R] - /** given the ldes stream URL, crawl the nodes of that stream */ - def crawl(stream: RDF.URI[R]): F[fs2.Stream[F, fs2.Chunk[UriNGraph[R]]]] = - import cats.syntax.all.toFunctorOps - import cats.syntax.flatMap.toFlatMapOps - for - visitedRef <- Ref.of[F, Set[RDF.URI[R]]](Set()) - views <- www.getPNG(stream).map(_.rel(tree.view)) - yield crawlNodesForward(stream, views, visitedRef) - - /* This crawls pages forward and collects all tree.member observations... - * This is very much tied to a particular use of ldes - * @returns an fs2.Stream of such observation mini graphs - **/ - def crawlNodesForward( - stream: RDF.URI[R], - startNodes: Seq[PNGraph[R]], - visitedRef: Ref[F, Set[RDF.URI[R]]] - ): fs2.Stream[F, fs2.Chunk[UriNGraph[R]]] = - import cats.syntax.all.toFlatMapOps - fs2.Stream.unfoldLoopEval(startNodes) { nodes => - import cats.syntax.traverse.{*, given} + /** given the ldes stream URL, crawl the nodes of that stream */ + def crawl(stream: RDF.URI[R]): F[fs2.Stream[F, fs2.Chunk[UriNGraph[R]]]] = import cats.syntax.all.toFunctorOps + import cats.syntax.flatMap.toFlatMapOps for - v <- visitedRef.get - // here we make sure we don't visit the same page twice and we don't fail on missing pages - pagesEither <- nodes.collect { - case ung: UriNGraph[R] if !v.contains(ung.point.fragmentLess) => - import cats.implicits.catsSyntaxApplicativeError - ung.jump[F].attempt - }.sequence - pages: Seq[UriNGraph[R]] = pagesEither.collect { case Right(png) => png } - _ <- visitedRef.update { v => - val urls = pages.map(_.name) - v.union(urls.toSet) - } - yield - val nextPages: Seq[UriNGraph[R]] = pages - .rel(tree.relation) - .filterType(tree.GreaterThanRelation) - .rel(tree.node) - .collect { case ung: UriNGraph[R] => ung } - // we need to place the pointer on the Collection of each page - ( - fs2.Chunk(pages*), - if nextPages.isEmpty then None - else Some(nextPages) - ) - } - end crawlNodesForward + visitedRef <- Ref.of[F, Set[RDF.URI[R]]](Set()) + views <- www.getPNG(stream).map(_.rel(tree.view)) + yield crawlNodesForward(stream, views, visitedRef) + + /* This crawls pages forward and collects all tree.member observations... + * This is very much tied to a particular use of ldes + * @returns an fs2.Stream of such observation mini graphs + **/ + def crawlNodesForward( + stream: RDF.URI[R], + startNodes: Seq[PNGraph[R]], + visitedRef: Ref[F, Set[RDF.URI[R]]] + ): fs2.Stream[F, fs2.Chunk[UriNGraph[R]]] = + import cats.syntax.all.toFlatMapOps + fs2.Stream.unfoldLoopEval(startNodes) { nodes => + import cats.syntax.traverse.{*, given} + import cats.syntax.all.toFunctorOps + for + v <- visitedRef.get + // here we make sure we don't visit the same page twice and we don't fail on missing pages + pagesEither <- nodes.collect { + case ung: UriNGraph[R] if !v.contains(ung.point.fragmentLess) => + import cats.implicits.catsSyntaxApplicativeError + ung.jump[F].attempt + }.sequence + pages: Seq[UriNGraph[R]] = pagesEither.collect { case Right(png) => png } + _ <- visitedRef.update { v => + val urls = pages.map(_.name) + v.union(urls.toSet) + } + yield + val nextPages: Seq[UriNGraph[R]] = pages.rel(tree.relation) + .filterType(tree.GreaterThanRelation).rel(tree.node) + .collect { case ung: UriNGraph[R] => ung } + // we need to place the pointer on the Collection of each page + ( + fs2.Chunk(pages*), + if nextPages.isEmpty then None + else Some(nextPages) + ) + } + end crawlNodesForward diff --git a/ldes/shared/src/main/scala/run/cosy/ldes/prefix/LDES.scala b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/LDES.scala index ae36007..d9cabf5 100644 --- a/ldes/shared/src/main/scala/run/cosy/ldes/prefix/LDES.scala +++ b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/LDES.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,20 +19,20 @@ package run.cosy.ldes.prefix import org.w3.banana.{Ops, PrefixBuilder, RDF} object LDES: - def apply[Rdf <: RDF](using ops: Ops[Rdf]) = new LDES() + def apply[Rdf <: RDF](using ops: Ops[Rdf]) = new LDES() class LDES[Rdf <: RDF](using ops: Ops[Rdf]) extends PrefixBuilder[Rdf]("ldes", ops.URI("https://w3id.org/ldes#")): - val EventStream = apply("EventStream") - val EventSource = apply("EventSource") - val RetentionPolicy = apply("RetentionPolicy") - val LatestVersionSubset = apply("LatestVersionSubset") - val DurationAgoPolicy = apply("DurationAgoPolicy") - val retentionPolicy = apply("retentionPolicy") - val amount = apply("amount") - val versionKey = apply("versionKey") - val versionOfPath = apply("versionOfPath") - val timestampPath = apply("timestampPath") - val versionMaterializationOf = apply("versionMaterializationOf") - val versionMaterializationUntil = apply("versionMaterializationUntil") + val EventStream = apply("EventStream") + val EventSource = apply("EventSource") + val RetentionPolicy = apply("RetentionPolicy") + val LatestVersionSubset = apply("LatestVersionSubset") + val DurationAgoPolicy = apply("DurationAgoPolicy") + val retentionPolicy = apply("retentionPolicy") + val amount = apply("amount") + val versionKey = apply("versionKey") + val versionOfPath = apply("versionOfPath") + val timestampPath = apply("timestampPath") + val versionMaterializationOf = apply("versionMaterializationOf") + val versionMaterializationUntil = apply("versionMaterializationUntil") diff --git a/ldes/shared/src/main/scala/run/cosy/ldes/prefix/SOSA.scala b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/SOSA.scala index 2a7b9d3..52cd59f 100644 --- a/ldes/shared/src/main/scala/run/cosy/ldes/prefix/SOSA.scala +++ b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/SOSA.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,44 +19,44 @@ package run.cosy.ldes.prefix import org.w3.banana.{Ops, PrefixBuilder, RDF} object SOSA: - def apply[Rdf <: RDF](using ops: Ops[Rdf]) = new SOSA() + def apply[Rdf <: RDF](using ops: Ops[Rdf]) = new SOSA() class SOSA[Rdf <: RDF](using ops: Ops[Rdf]) extends PrefixBuilder[Rdf]("tree", ops.URI("http://www.w3.org/ns/sosa/")): - val FeatureOfInterest = apply("FeatureOfInterest") - val ObservableProperty = apply("ObservableProperty") - val ActuatableProperty = apply("ActuatableProperty") - val Sample = apply("Sample") - val hasSample = apply("hasSample") - val isSampleOf = apply("isSampleOf") - val Platform = apply("Platform") - val hosts = apply("hosts") - val isHostedBy = apply("isHostedBy") - val Procedure = apply("Procedure") - val Sensor = apply("Sensor") - val observes = apply("observes") - val isObservedBy = apply("isObservedBy") - val Actuator = apply("Actuator") - val Sampler = apply("Sampler") - val usedProcedure = apply("usedProcedure") - val hasFeatureOfInterest = apply("hasFeatureOfInterest") - val isFeatureOfInterestOf = apply("isFeatureOfInterestOf") - val Observation = apply("Observation") - val madeObservation = apply("madeObservation") - val madeBySensor = apply("madeBySensor") - val observedProperty = apply("observedProperty") - val Actuation = apply("Actuation") - val madeActuation = apply("madeActuation") - val madeByActuator = apply("madeByActuator") - val actsOnProperty = apply("actsOnProperty") - val isActedOnBy = apply("isActedOnBy") - val Sampling = apply("Sampling") - val madeSampling = apply("madeSampling") - val madeBySampler = apply("madeBySampler") - val Result = apply("Result") - val hasResult = apply("hasResult") - val isResultOf = apply("isResultOf") - val hasSimpleResult = apply("hasSimpleResult") - val resultTime = apply("resultTime") - val phenomenonTime = apply("phenomenonTime") + val FeatureOfInterest = apply("FeatureOfInterest") + val ObservableProperty = apply("ObservableProperty") + val ActuatableProperty = apply("ActuatableProperty") + val Sample = apply("Sample") + val hasSample = apply("hasSample") + val isSampleOf = apply("isSampleOf") + val Platform = apply("Platform") + val hosts = apply("hosts") + val isHostedBy = apply("isHostedBy") + val Procedure = apply("Procedure") + val Sensor = apply("Sensor") + val observes = apply("observes") + val isObservedBy = apply("isObservedBy") + val Actuator = apply("Actuator") + val Sampler = apply("Sampler") + val usedProcedure = apply("usedProcedure") + val hasFeatureOfInterest = apply("hasFeatureOfInterest") + val isFeatureOfInterestOf = apply("isFeatureOfInterestOf") + val Observation = apply("Observation") + val madeObservation = apply("madeObservation") + val madeBySensor = apply("madeBySensor") + val observedProperty = apply("observedProperty") + val Actuation = apply("Actuation") + val madeActuation = apply("madeActuation") + val madeByActuator = apply("madeByActuator") + val actsOnProperty = apply("actsOnProperty") + val isActedOnBy = apply("isActedOnBy") + val Sampling = apply("Sampling") + val madeSampling = apply("madeSampling") + val madeBySampler = apply("madeBySampler") + val Result = apply("Result") + val hasResult = apply("hasResult") + val isResultOf = apply("isResultOf") + val hasSimpleResult = apply("hasSimpleResult") + val resultTime = apply("resultTime") + val phenomenonTime = apply("phenomenonTime") diff --git a/ldes/shared/src/main/scala/run/cosy/ldes/prefix/TREE.scala b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/TREE.scala index 6b4f6d0..cff4cdd 100644 --- a/ldes/shared/src/main/scala/run/cosy/ldes/prefix/TREE.scala +++ b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/TREE.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,38 +19,38 @@ package run.cosy.ldes.prefix import org.w3.banana.{Ops, PrefixBuilder, RDF} object TREE: - def apply[Rdf <: RDF](using ops: Ops[Rdf]) = new TREE() + def apply[Rdf <: RDF](using ops: Ops[Rdf]) = new TREE() class TREE[Rdf <: RDF](using ops: Ops[Rdf]) extends PrefixBuilder[Rdf]("tree", ops.URI("https://w3id.org/tree#")): - val Collection = apply("Collection") - val ViewDescription = apply("ViewDescription") - val Node = apply("Node") - val Relation = apply("Relation") - val ConditionalImport = apply("ConditionalImport") - val PrefixRelation = apply("PrefixRelation") - val SubstringRelation = apply("SubstringRelation") - val SuffixRelation = apply("SuffixRelation") - val GreaterThanRelation = apply("GreaterThanRelation") - val GreaterThanOrEqualToRelation = apply("GreaterThanOrEqualToRelation") - val LessThanRelation = apply("LessThanRelation") - val LessThanOrEqualToRelation = apply("LessThanOrEqualToRelation") - val EqualToRelation = apply("EqualToRelation") - val GeospatiallyContainsRelation = apply("GeospatiallyContainsRelation") - val InBetweenRelation = apply("InBetweenRelation") - val viewDescription = apply("viewDescription") - val relation = apply("relation") - val remainingItems = apply("remainingItems") - val node = apply("node") - val value = apply("value") - val path = apply("path") - val view = apply("view") - val member = apply("member") - val search = apply("search") - val shape = apply("shape") - val conditionalImport = apply("conditionalImport") - val zoom = apply("zoom") - val longitudeTile = apply("longitudeTile") - val latitudeTile = apply("latitudeTile") - val timeQuery = apply("timeQuery") + val Collection = apply("Collection") + val ViewDescription = apply("ViewDescription") + val Node = apply("Node") + val Relation = apply("Relation") + val ConditionalImport = apply("ConditionalImport") + val PrefixRelation = apply("PrefixRelation") + val SubstringRelation = apply("SubstringRelation") + val SuffixRelation = apply("SuffixRelation") + val GreaterThanRelation = apply("GreaterThanRelation") + val GreaterThanOrEqualToRelation = apply("GreaterThanOrEqualToRelation") + val LessThanRelation = apply("LessThanRelation") + val LessThanOrEqualToRelation = apply("LessThanOrEqualToRelation") + val EqualToRelation = apply("EqualToRelation") + val GeospatiallyContainsRelation = apply("GeospatiallyContainsRelation") + val InBetweenRelation = apply("InBetweenRelation") + val viewDescription = apply("viewDescription") + val relation = apply("relation") + val remainingItems = apply("remainingItems") + val node = apply("node") + val value = apply("value") + val path = apply("path") + val view = apply("view") + val member = apply("member") + val search = apply("search") + val shape = apply("shape") + val conditionalImport = apply("conditionalImport") + val zoom = apply("zoom") + val longitudeTile = apply("longitudeTile") + val latitudeTile = apply("latitudeTile") + val timeQuery = apply("timeQuery") diff --git a/ldes/shared/src/main/scala/run/cosy/ldes/prefix/WGS84.scala b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/WGS84.scala index a63ac10..40707b0 100644 --- a/ldes/shared/src/main/scala/run/cosy/ldes/prefix/WGS84.scala +++ b/ldes/shared/src/main/scala/run/cosy/ldes/prefix/WGS84.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,14 +19,14 @@ package run.cosy.ldes.prefix import org.w3.banana.{Ops, PrefixBuilder, RDF} object WGS84: - def apply[R <: RDF](using Ops[R]) = new WGS84() + def apply[R <: RDF](using Ops[R]) = new WGS84() class WGS84[R <: RDF](using ops: Ops[R]) extends PrefixBuilder[R]("wgs84", ops.URI("http://www.w3.org/2003/01/geo/wgs84_pos#")): - lazy val Point = apply("Point") - lazy val SpatialThing = apply("SpatialThing") - lazy val alt = apply("alt") - lazy val lat = apply("lat") - lazy val lat_long = apply("lat_long") - lazy val location = apply("location") + lazy val Point = apply("Point") + lazy val SpatialThing = apply("SpatialThing") + lazy val alt = apply("alt") + lazy val lat = apply("lat") + lazy val lat_long = apply("lat_long") + lazy val location = apply("location") diff --git a/ldes/shared/src/test/scala/run/cosy/ld/FoafWebTest.scala b/ldes/shared/src/test/scala/run/cosy/ld/FoafWebTest.scala index 163c7a2..d9b6cc1 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/FoafWebTest.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/FoafWebTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,92 +23,81 @@ import org.w3.banana.Ops import munit.CatsEffectSuite import run.cosy.ld.MiniFoafWWW.EricP -trait FoafWebTest[R <: RDF]()(using ops: Ops[R]) extends CatsEffectSuite { - val miniWeb = new MiniFoafWWW[R] - import miniWeb.foaf - given www: Web[IO, R] = miniWeb - import ops.{*, given} - import PNGraph.{*, given} - import MiniFoafWWW.* - import cats.effect.IO.asyncForIO +trait FoafWebTest[R <: RDF]()(using ops: Ops[R]) extends CatsEffectSuite: + val miniWeb = new MiniFoafWWW[R] + import miniWeb.foaf + given www: Web[IO, R] = miniWeb + import ops.{*, given} + import PNGraph.{*, given} + import MiniFoafWWW.* + import cats.effect.IO.asyncForIO - test("find bbl friends WebIDs inside graph (no jump)") { + test("find bbl friends WebIDs inside graph (no jump)") { - val bblng: IO[UriNGraph[R]] = www.getPNG(URI(Bbl)) - bblng.map { bblNg => - // todo: make it possible to just get nodes - val kns: Seq[String] = (bblNg / foaf.knows).collect { case u: UriNGraph[R] => - u.point.value - } - assertEquals(Set(kns*), Set(EricP, Timbl, CSarven)) - } + val bblng: IO[UriNGraph[R]] = www.getPNG(URI(Bbl)) + bblng.map { bblNg => + // todo: make it possible to just get nodes + val kns: Seq[String] = (bblNg / foaf.knows).collect { case u: UriNGraph[R] => + u.point.value + } + assertEquals(Set(kns*), Set(EricP, Timbl, CSarven)) + } - } + } - test("find bblFriend WebIDs after jumping") { - www.getPNG(URI(Bbl)).flatMap { bblNg => - val friendsDef: fs2.Stream[IO, PNGraph[R]] = (bblNg / foaf.knows).jump - val expectedURIs: Set[String] = Set(EricP, Timbl, CSarven) - val result1: IO[Set[String]] = friendsDef.compile.toList.map(pnglst => - val uris: Seq[String] = pnglst.collect { case u: UriNGraph[R] => u.point.value } - Set(uris*) - ) - result1.map(set => assertEquals(set, expectedURIs)) - } + test("find bblFriend WebIDs after jumping") { + www.getPNG(URI(Bbl)).flatMap { bblNg => + val friendsDef: fs2.Stream[IO, PNGraph[R]] = (bblNg / foaf.knows).jump + val expectedURIs: Set[String] = Set(EricP, Timbl, CSarven) + val result1: IO[Set[String]] = friendsDef.compile.toList.map(pnglst => + val uris: Seq[String] = pnglst.collect { case u: UriNGraph[R] => u.point.value } + Set(uris*) + ) + result1.map(set => assertEquals(set, expectedURIs)) + } - // todo: test what happens if a WebID is broken. - } + // todo: test what happens if a WebID is broken. + } - test("find canonical names of friends (as defined in their profile)") { - www.getPNG(URI(Bbl)).flatMap { bblNg => - // most of the names are only available from the definitional graphs - val names: fs2.Stream[IO, String] = (bblNg / foaf.knows).jump - .collect { case ug: UriNGraph[R] => - fs2.Stream(ug / foaf.name*) - } - .flatten - .collect { case litG: LiteralNGraph[R] => litG.point.text } - names.compile.toList.map(lst => - assertEquals(Set(lst*), Set("Tim Berners-Lee", "Eric Prud'hommeaux", "Sarven Capadisli")) - ) - } - } + test("find canonical names of friends (as defined in their profile)") { + www.getPNG(URI(Bbl)).flatMap { bblNg => + // most of the names are only available from the definitional graphs + val names: fs2.Stream[IO, String] = (bblNg / foaf.knows).jump + .collect { case ug: UriNGraph[R] => fs2.Stream(ug / foaf.name*) }.flatten + .collect { case litG: LiteralNGraph[R] => litG.point.text } + names.compile.toList.map(lst => + assertEquals(Set(lst*), Set("Tim Berners-Lee", "Eric Prud'hommeaux", "Sarven Capadisli")) + ) + } + } - test("find all names of friends (local and remote)") { - www.getPNG(URI(Bbl)).flatMap { bblNg => - // most of the names are only available from the definitional graphs - val allNames: fs2.Stream[IO, String] = (bblNg / foaf.knows).jump - .collect { case ug: SubjPNGraph[R] => - fs2.Stream(ug / foaf.name*) - } - .flatten - .collect { case litG: LiteralNGraph[R] => litG.point.text } + test("find all names of friends (local and remote)") { + www.getPNG(URI(Bbl)).flatMap { bblNg => + // most of the names are only available from the definitional graphs + val allNames: fs2.Stream[IO, String] = (bblNg / foaf.knows).jump + .collect { case ug: SubjPNGraph[R] => fs2.Stream(ug / foaf.name*) }.flatten + .collect { case litG: LiteralNGraph[R] => litG.point.text } - allNames.compile.toList.map { lst => - assertEquals( - Set(lst*), - Set("Tim Berners-Lee", "Eric Prud'hommeaux", "Sarven Capadisli", "James Gosling") - ) - } - } - } + allNames.compile.toList.map { lst => + assertEquals( + Set(lst*), + Set("Tim Berners-Lee", "Eric Prud'hommeaux", "Sarven Capadisli", "James Gosling") + ) + } + } + } - test("find all names of friends (local and remote) - shorter version") { - www.getPNG(URI(Bbl)).flatMap { bblNg => - // most of the names are only available from the definitional graphs - val allNames: fs2.Stream[IO, String] = - bblNg - .rel(foaf.knows) - .jump - .rel(foaf.name) + test("find all names of friends (local and remote) - shorter version") { + www.getPNG(URI(Bbl)).flatMap { bblNg => + // most of the names are only available from the definitional graphs + val allNames: fs2.Stream[IO, String] = bblNg.rel(foaf.knows).jump.rel(foaf.name) .collect { case litG: LiteralNGraph[R] => litG.point.text } - allNames.compile.toList.map { lst => - assertEquals( - Set(lst*), - Set("Tim Berners-Lee", "Eric Prud'hommeaux", "Sarven Capadisli", "James Gosling") - ) - } - } - } -} + allNames.compile.toList.map { lst => + assertEquals( + Set(lst*), + Set("Tim Berners-Lee", "Eric Prud'hommeaux", "Sarven Capadisli", "James Gosling") + ) + } + } + } diff --git a/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala b/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala index 87755ed..65d8a2d 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/JenaWebTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ldes/shared/src/test/scala/run/cosy/ld/LdesBrokenWebTest.scala b/ldes/shared/src/test/scala/run/cosy/ld/LdesBrokenWebTest.scala index 78bd9d5..7438382 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/LdesBrokenWebTest.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/LdesBrokenWebTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,79 +26,71 @@ import run.cosy.ld.ldes.{BrokenMiniLdesWWW, MiniLdesWWW} import scala.collection.immutable.{Seq, Set} trait LdesBrokenWebTest[R <: RDF]()(using ops: Ops[R]) extends CatsEffectSuite: - import MiniLdesWWW.* - val miniWeb = new BrokenMiniLdesWWW[R] - given www: Web[IO, R] = miniWeb - import miniWeb.{sosa, tree, wgs84} - import ops.{*, given} - import run.cosy.ld.PNGraph.* + import MiniLdesWWW.* + val miniWeb = new BrokenMiniLdesWWW[R] + given www: Web[IO, R] = miniWeb + import miniWeb.{sosa, tree, wgs84} + import ops.{*, given} + import run.cosy.ld.PNGraph.* - test("get circular data with links going nowhere without breaking") { - import cats.syntax.traverse.{*, given} - val z: IO[fs2.Stream[IO, RDF.Graph[R]]] = - for - visitedRef <- Ref.of[IO, Set[RDF.URI[R]]](Set()) - views <- www.getPNG(URI(Collection)).map(_.rel(tree.view)) - yield fs2.Stream.unfoldLoopEval(views) { views => - import cats.syntax.traverse.{*, given} - for - v <- visitedRef.get - // here we make sure we don't visit the same page twice and we don't fail on missing pages - pagesEither <- views.collect { - case ung: UriNGraph[R] if !v.contains(ung.point.fragmentLess) => - ung.jump[IO].attempt - }.sequence - pages = pagesEither.collect { case Right(png) => png } - _ <- visitedRef.update { v => - val urls = pages.map(_.name) - v.union(urls.toSet) - } - yield - val nextPages: Seq[UriNGraph[R]] = pages - .rel(tree.relation) - .filterType(tree.GreaterThanRelation) - .rel(tree.node) - .collect { case ung: UriNGraph[R] => ung } - // we need to place the pointer on the Collection of each page - val collInPages: Seq[UriNGraph[R]] = - val uc = URI(Collection) - pages.map(ung => new UriNGraph[R](uc, ung.name, ung.graph)) - val obs: RDF.Graph[R] = collInPages - .rel(tree.member) - .map(png => - png.collect( - rdf.typ, - wgs84.location, - sosa.hasSimpleResult, - sosa.madeBySensor, - sosa.observedProperty, - sosa.resultTime - )() - ) - .fold(Graph.empty)((g1, g2) => g1 union g2) - ( - obs, - if nextPages.isEmpty then None - else Some(nextPages) - ) - } + test("get circular data with links going nowhere without breaking") { + import cats.syntax.traverse.{*, given} + val z: IO[fs2.Stream[IO, RDF.Graph[R]]] = + for + visitedRef <- Ref.of[IO, Set[RDF.URI[R]]](Set()) + views <- www.getPNG(URI(Collection)).map(_.rel(tree.view)) + yield fs2.Stream.unfoldLoopEval(views) { views => + import cats.syntax.traverse.{*, given} + for + v <- visitedRef.get + // here we make sure we don't visit the same page twice and we don't fail on missing pages + pagesEither <- views.collect { + case ung: UriNGraph[R] if !v.contains(ung.point.fragmentLess) => ung.jump[IO].attempt + }.sequence + pages = pagesEither.collect { case Right(png) => png } + _ <- visitedRef.update { v => + val urls = pages.map(_.name) + v.union(urls.toSet) + } + yield + val nextPages: Seq[UriNGraph[R]] = pages.rel(tree.relation) + .filterType(tree.GreaterThanRelation).rel(tree.node) + .collect { case ung: UriNGraph[R] => ung } + // we need to place the pointer on the Collection of each page + val collInPages: Seq[UriNGraph[R]] = + val uc = URI(Collection) + pages.map(ung => new UriNGraph[R](uc, ung.name, ung.graph)) + val obs: RDF.Graph[R] = collInPages.rel(tree.member).map(png => + png.collect( + rdf.typ, + wgs84.location, + sosa.hasSimpleResult, + sosa.madeBySensor, + sosa.observedProperty, + sosa.resultTime + )() + ).fold(Graph.empty)((g1, g2) => g1 union g2) + ( + obs, + if nextPages.isEmpty then None + else Some(nextPages) + ) + } - z.flatMap { graphs => - graphs.compile.toList.map { lst => - val g: RDF.Graph[R] = lst.fold(Graph.empty)((g1, g2) => g1 union g2) - val gTrpls: Set[RDF.Triple[R]] = Set(g.triples.toSeq*) - val expectedTrpls: Seq[RDF.Triple[R]] = miniWeb.obsrvs.iterator - .map { (uristr, rtriples) => - val u = ll.uri.AbsoluteUrl.parse(uristr) - rtriples.flatten.map((rt: RDF.rTriple[R]) => rt.resolveAgainst(u)._1) - } - .toSeq - .flatten - val expected: Set[RDF.Triple[R]] = Set(expectedTrpls*) - // we can compare them as sets as we have no blank nodes - assertEquals(expected.diff(gTrpls), Set.empty) - assertEquals(gTrpls.size, expected.size) - assertEquals(gTrpls, expected) - } - } - } + z.flatMap { graphs => + graphs.compile.toList.map { lst => + val g: RDF.Graph[R] = lst.fold(Graph.empty)((g1, g2) => g1 union g2) + val gTrpls: Set[RDF.Triple[R]] = Set(g.triples.toSeq*) + val expectedTrpls: Seq[RDF.Triple[R]] = miniWeb.obsrvs.iterator + .map { (uristr, rtriples) => + val u = ll.uri.AbsoluteUrl.parse(uristr) + rtriples.flatten.map((rt: RDF.rTriple[R]) => rt.resolveAgainst(u)._1) + }.toSeq.flatten + val expected: Set[RDF.Triple[R]] = Set(expectedTrpls*) + // we can compare them as sets as we have no blank nodes + assertEquals(expected.diff(gTrpls), Set.empty) + assertEquals(gTrpls.size, expected.size) + assertEquals(gTrpls, expected) + } + } + } diff --git a/ldes/shared/src/test/scala/run/cosy/ld/LdesSimpleWebTest.scala b/ldes/shared/src/test/scala/run/cosy/ld/LdesSimpleWebTest.scala index 2491c95..c788177 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/LdesSimpleWebTest.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/LdesSimpleWebTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,50 +28,44 @@ import run.cosy.ld.ldes.MiniLdesWWW * work on that in separate tests. Here we just want to test how to get the data */ trait LdesSimpleWebTest[R <: RDF]()(using ops: Ops[R]) extends CatsEffectSuite: - import MiniLdesWWW.* - val miniWeb = new MiniLdesWWW[R] - given www: Web[IO, R] = miniWeb - import miniWeb.{sosa, tree, wgs84} - import ops.{*, given} - import run.cosy.ld.PNGraph.* + import MiniLdesWWW.* + val miniWeb = new MiniLdesWWW[R] + given www: Web[IO, R] = miniWeb + import miniWeb.{sosa, tree, wgs84} + import ops.{*, given} + import run.cosy.ld.PNGraph.* - test("walk through pages starting from first page") { - // we start from the container url, and jump to the first page - val page1_IO: IO[UriNGraph[R]] = www.getPNG(URI(D09_05)) + test("walk through pages starting from first page") { + // we start from the container url, and jump to the first page + val page1_IO: IO[UriNGraph[R]] = www.getPNG(URI(D09_05)) - // nodeStr has to be in the right graph - def next(node: SubjPNGraph[R]): Seq[PNGraph[R]] = - node - .rel(tree.relation) - .filterType(tree.GreaterThanRelation) - .rel(tree.node) + // nodeStr has to be in the right graph + def next(node: SubjPNGraph[R]): Seq[PNGraph[R]] = node.rel(tree.relation) + .filterType(tree.GreaterThanRelation).rel(tree.node) - val result: IO[fs2.Stream[IO, String]] = - for page1 <- page1_IO - yield - // this is a bit clumsy but it works - fs2.Stream(page1.name.value) - ++ fs2.Stream.unfoldEval(page1) { (node: UriNGraph[R]) => - val ns: Seq[PNGraph[R]] = next(node) - val x: Option[IO[UriNGraph[R]]] = ns.collectFirst { case ui: UriNGraph[R] => - ui.jump - } - x match - case None => IO(None) - case Some(ioNgr) => ioNgr.map(ngr => Some((ngr.name.value, ngr))) - } - result.flatMap(names => - names.compile.toList.map { lst => - assertEquals(lst.toSet, Set(D09_05, D09_06, D09_07)) - } - ) - } + val result: IO[fs2.Stream[IO, String]] = + for page1 <- page1_IO + yield + // this is a bit clumsy but it works + fs2.Stream(page1.name.value) + ++ fs2.Stream.unfoldEval(page1) { (node: UriNGraph[R]) => + val ns: Seq[PNGraph[R]] = next(node) + val x: Option[IO[UriNGraph[R]]] = ns.collectFirst { case ui: UriNGraph[R] => ui.jump } + x match + case None => IO(None) + case Some(ioNgr) => ioNgr.map(ngr => Some((ngr.name.value, ngr))) + } + result.flatMap(names => + names.compile.toList.map { lst => + assertEquals(lst.toSet, Set(D09_05, D09_06, D09_07)) + } + ) + } - test("walk through pages but starting from Collection") { - val z: IO[fs2.Stream[IO, String]] = - for views <- www.getPNG(URI(Collection)).map(_.rel(tree.view)) - yield fs2.Stream - .unfoldLoopEval(views) { (views: Seq[PNGraph[R]]) => + test("walk through pages but starting from Collection") { + val z: IO[fs2.Stream[IO, String]] = + for views <- www.getPNG(URI(Collection)).map(_.rel(tree.view)) + yield fs2.Stream.unfoldLoopEval(views) { (views: Seq[PNGraph[R]]) => import cats.syntax.traverse.{*, given} val x: Seq[IO[UriNGraph[R]]] = // because we are using unfoldLoopEval we need to work with the IO effect, not Streams @@ -79,88 +73,75 @@ trait LdesSimpleWebTest[R <: RDF]()(using ops: Ops[R]) extends CatsEffectSuite: views.collect { case ung: UriNGraph[R] => ung.jump } val y: IO[Seq[UriNGraph[R]]] = x.sequence val res: IO[(Chunk[String], Option[Seq[UriNGraph[R]]])] = y.map { seqUng => - val s: Seq[UriNGraph[R]] = seqUng - .rel(tree.relation) - .filterType(tree.GreaterThanRelation) - .rel(tree.node) - .collect { case ung: UriNGraph[R] => ung } - ( - Chunk.seq(views.collect { case p: UriNGraph[R] => p.point.value }), - if s.isEmpty then None - else Some(s) - ) + val s: Seq[UriNGraph[R]] = seqUng.rel(tree.relation) + .filterType(tree.GreaterThanRelation).rel(tree.node) + .collect { case ung: UriNGraph[R] => ung } + ( + Chunk.seq(views.collect { case p: UriNGraph[R] => p.point.value }), + if s.isEmpty then None + else Some(s) + ) } res - } - .unchunks - z.flatMap { urls => - urls.compile.toList.map { lst => - assertEquals(lst.toSet, Set(D09_05, D09_06, D09_07)) - } - } - } + }.unchunks + z.flatMap { urls => + urls.compile.toList.map { lst => + assertEquals(lst.toSet, Set(D09_05, D09_06, D09_07)) + } + } + } - test("walk through pages and collect observations") { - val z: IO[fs2.Stream[IO, RDF.Graph[R]]] = - for views <- www.getPNG(URI(Collection)).map(_.rel(tree.view)) - yield fs2.Stream - .unfoldLoopEval(views) { (views: Seq[PNGraph[R]]) => + test("walk through pages and collect observations") { + val z: IO[fs2.Stream[IO, RDF.Graph[R]]] = + for views <- www.getPNG(URI(Collection)).map(_.rel(tree.view)) + yield fs2.Stream.unfoldLoopEval(views) { (views: Seq[PNGraph[R]]) => import cats.syntax.traverse.{*, given} - val x: Seq[IO[UriNGraph[R]]] = - views.collect { case ung: UriNGraph[R] => ung.jump } + val x: Seq[IO[UriNGraph[R]]] = views.collect { case ung: UriNGraph[R] => ung.jump } // note: a problem with x.sequence is that it would need all UriNGraphs to be complete // before the IO is complete. What if some get stuck? val pagesIO: IO[Seq[UriNGraph[R]]] = x.sequence // val pagesIO: IO[Seq[UriNGraph[R]]] = // views.collect({ case ung: UriNGraph[R] => ung.jump }).sequence val res: IO[(RDF.Graph[R], Option[Seq[UriNGraph[R]]])] = pagesIO.map { seqUng => - val nextPages: Seq[UriNGraph[R]] = - seqUng - .rel(tree.relation) - .filterType(tree.GreaterThanRelation) - .rel(tree.node) - .collect { case ung: UriNGraph[R] => ung } - // we need to place the pointer on the Collection of each page - val collInPages: Seq[UriNGraph[R]] = - val uc = URI(Collection) - seqUng.map(ung => new UriNGraph[R](uc, ung.name, ung.graph)) - val obs = collInPages - .rel(tree.member) - .map(png => - png.collect( - rdf.typ, - wgs84.location, - sosa.hasSimpleResult, - sosa.madeBySensor, - sosa.observedProperty, - sosa.resultTime - )() - ) - .fold(Graph.empty)((g1, g2) => g1 union g2) - ( - obs, - if nextPages.isEmpty then None - else Some(nextPages) - ) + val nextPages: Seq[UriNGraph[R]] = seqUng.rel(tree.relation) + .filterType(tree.GreaterThanRelation).rel(tree.node) + .collect { case ung: UriNGraph[R] => ung } + // we need to place the pointer on the Collection of each page + val collInPages: Seq[UriNGraph[R]] = + val uc = URI(Collection) + seqUng.map(ung => new UriNGraph[R](uc, ung.name, ung.graph)) + val obs = collInPages.rel(tree.member).map(png => + png.collect( + rdf.typ, + wgs84.location, + sosa.hasSimpleResult, + sosa.madeBySensor, + sosa.observedProperty, + sosa.resultTime + )() + ).fold(Graph.empty)((g1, g2) => g1 union g2) + ( + obs, + if nextPages.isEmpty then None + else Some(nextPages) + ) } res - } - z.flatMap { graphs => - graphs.compile.toList.map { lst => - val g: RDF.Graph[R] = lst.fold(Graph.empty)((g1, g2) => g1 union g2) - val gTrpls: Set[RDF.Triple[R]] = Set(g.triples.toSeq*) - val expectedTrpls: Seq[RDF.Triple[R]] = miniWeb.obsrvs.iterator - .map { (uristr, rtriples) => - val u = ll.uri.AbsoluteUrl.parse(uristr) - rtriples.flatten.map((rt: RDF.rTriple[R]) => rt.resolveAgainst(u)._1) - } - .toSeq - .flatten - val expected: Set[RDF.Triple[R]] = Set(expectedTrpls*) - // we can compare them as sets as we have no blank nodes - assertEquals(expected.diff(gTrpls), Set.empty) - assertEquals(gTrpls.size, expected.size) - assertEquals(gTrpls, expected) - } - } - } + } + z.flatMap { graphs => + graphs.compile.toList.map { lst => + val g: RDF.Graph[R] = lst.fold(Graph.empty)((g1, g2) => g1 union g2) + val gTrpls: Set[RDF.Triple[R]] = Set(g.triples.toSeq*) + val expectedTrpls: Seq[RDF.Triple[R]] = miniWeb.obsrvs.iterator + .map { (uristr, rtriples) => + val u = ll.uri.AbsoluteUrl.parse(uristr) + rtriples.flatten.map((rt: RDF.rTriple[R]) => rt.resolveAgainst(u)._1) + }.toSeq.flatten + val expected: Set[RDF.Triple[R]] = Set(expectedTrpls*) + // we can compare them as sets as we have no blank nodes + assertEquals(expected.diff(gTrpls), Set.empty) + assertEquals(gTrpls.size, expected.size) + assertEquals(gTrpls, expected) + } + } + } diff --git a/ldes/shared/src/test/scala/run/cosy/ld/MiniFoafWWW.scala b/ldes/shared/src/test/scala/run/cosy/ld/MiniFoafWWW.scala index 0a0a4ed..0413d32 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/MiniFoafWWW.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/MiniFoafWWW.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,55 +25,50 @@ import org.w3.banana.diesel import org.w3.banana.diesel.{*, given} object MiniFoafWWW: - val BblCard = "https://bblfish.net/people/henry/card" - val Bbl = BblCard + "#me" - val TimblCard = "https://www.w3.org/People/Berners-Lee/card" - val Timbl = TimblCard + "#i" - val EricPCard = "https://www.w3.org/People/Eric/ericP-foaf.rdf" - val EricP = EricPCard + "#ericP" - val CSarvenCard = "https://csarven.ca/" - val CSarven = CSarvenCard + "#i" + val BblCard = "https://bblfish.net/people/henry/card" + val Bbl = BblCard + "#me" + val TimblCard = "https://www.w3.org/People/Berners-Lee/card" + val Timbl = TimblCard + "#i" + val EricPCard = "https://www.w3.org/People/Eric/ericP-foaf.rdf" + val EricP = EricPCard + "#ericP" + val CSarvenCard = "https://csarven.ca/" + val CSarven = CSarvenCard + "#i" class MiniFoafWWW[R <: RDF](using ops: Ops[R]) extends Web[IO, R]: - import ops.{*, given} - import MiniFoafWWW.* + import ops.{*, given} + import MiniFoafWWW.* - val foaf = prefix.FOAF[R] - val fr = Lang("fr") // todo, put together a list of Lang constants - val en = Lang("en") + val foaf = prefix.FOAF[R] + val fr = Lang("fr") // todo, put together a list of Lang constants + val en = Lang("en") - def getPNG(url: RDF.URI[R]): IO[UriNGraph[R]] = - val doc = url.fragmentLess - get(doc).map(g => new UriNGraph(url, doc, g)) + def getPNG(url: RDF.URI[R]): IO[UriNGraph[R]] = + val doc = url.fragmentLess + get(doc).map(g => new UriNGraph(url, doc, g)) - def get(url: RDF.URI[R]): IO[RDF.Graph[R]] = - import scala.language.implicitConversions - val res: RDF.rGraph[R] = - url.value match - case BblCard => - (rURI("#me") -- foaf.name ->- "Henry Story".lang(en) - -- foaf.knows ->- URI(EricP) - -- foaf.knows ->- URI(CSarven) - -- foaf.knows ->- URI(Timbl) - -- foaf.knows ->- (BNode() -- foaf.name ->- "James Gosling")).graph - case TimblCard => - (rURI("#i") -- foaf.name ->- "Tim Berners-Lee".lang(en) - -- foaf.knows ->- URI(Bbl) - -- foaf.knows ->- URI(CSarven) - -- foaf.knows ->- URI(EricP) - -- foaf.knows ->- (BNode() -- foaf.name ->- "Vint Cerf")).graph - case EricPCard => - (rURI("#ericP") -- foaf.name ->- "Eric Prud'hommeaux".lang(fr) - -- foaf.knows ->- URI(Bbl) - -- foaf.knows ->- URI(CSarven) - -- foaf.knows ->- URI(Timbl)).graph - case CSarvenCard => - (rURI("#i") -- foaf.name ->- "Sarven Capadisli".lang(en) - -- foaf.knows ->- URI(Bbl) - -- foaf.knows ->- URI(Timbl) - -- foaf.knows ->- URI(EricP)).graph + def get(url: RDF.URI[R]): IO[RDF.Graph[R]] = + import scala.language.implicitConversions + val res: RDF.rGraph[R] = url.value match + case BblCard => (rURI("#me") -- foaf.name ->- "Henry Story".lang(en) + -- foaf.knows ->- URI(EricP) + -- foaf.knows ->- URI(CSarven) + -- foaf.knows ->- URI(Timbl) + -- foaf.knows ->- (BNode() -- foaf.name ->- "James Gosling")).graph + case TimblCard => (rURI("#i") -- foaf.name ->- "Tim Berners-Lee".lang(en) + -- foaf.knows ->- URI(Bbl) + -- foaf.knows ->- URI(CSarven) + -- foaf.knows ->- URI(EricP) + -- foaf.knows ->- (BNode() -- foaf.name ->- "Vint Cerf")).graph + case EricPCard => (rURI("#ericP") -- foaf.name ->- "Eric Prud'hommeaux".lang(fr) + -- foaf.knows ->- URI(Bbl) + -- foaf.knows ->- URI(CSarven) + -- foaf.knows ->- URI(Timbl)).graph + case CSarvenCard => (rURI("#i") -- foaf.name ->- "Sarven Capadisli".lang(en) + -- foaf.knows ->- URI(Bbl) + -- foaf.knows ->- URI(Timbl) + -- foaf.knows ->- URI(EricP)).graph - case _ => ops.rGraph.empty + case _ => ops.rGraph.empty - IO(res.resolveAgainst(AbsoluteUrl.parse(url.value))) - end get + IO(res.resolveAgainst(AbsoluteUrl.parse(url.value))) + end get diff --git a/ldes/shared/src/test/scala/run/cosy/ld/ldes/BrokenMiniLdesWWW.scala b/ldes/shared/src/test/scala/run/cosy/ld/ldes/BrokenMiniLdesWWW.scala index dbc3524..ef6f95f 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/ldes/BrokenMiniLdesWWW.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/ldes/BrokenMiniLdesWWW.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,45 +22,46 @@ import org.w3.banana.diesel.{*, given} /** Ldes example with circularity and link pointing nowhere */ class BrokenMiniLdesWWW[R <: RDF](using ops: Ops[R]) extends MiniLdesWWW[R]: - import MiniLdesWWW.* - import ops.{*, given} + import MiniLdesWWW.* + import ops.{*, given} - // we introduce a circularity in the pagination of D09_07 - override def getRelativeGraph(url: RDF.URI[R]): Option[RDF.rGraph[R]] = - import scala.language.implicitConversions - url.value match - case D09_07 => - val g: RDF.rGraph[R] = (rURI("").a(tree.Node) - -- tree.relation ->- ( - BNode().a(tree.LessThanRelation) - -- tree.node ->- rURI("2021-09-07") - -- tree.path ->- sosa.resultTime - -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) - ) - -- tree.relation ->- ( - BNode().a(tree.GreaterThanRelation) - -- tree.node ->- rURI("2021-09-09") // warning: does not exist - -- tree.path ->- sosa.resultTime - -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) - ) - -- tree.relation ->- ( - BNode().a(tree.GreaterThanRelation) - -- tree.node ->- rURI("2021-09-05") // warning: error circularity - -- tree.path ->- sosa.resultTime - -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) - )).graph ++ obsrvs(D09_07).flatten ++ ( - rURI(".").a(ldes.EventStream) - -- ldes.timestampPath ->- sosa.resultTime - -- tree.shape ->- rURI("flows-shacl") - -- tree.view ->- rURI("") - -- tree.member ->- rURI("#658") - -- tree.member ->- rURI("#3074") - -- tree.member ->- rURI("#637") - ).graph.triples.toSeq - Some(g) - case Collection => - // we add a link to a non existing view - super.getRelativeGraph(url).map { rg => - rg + rTriple(rURI(""), tree.view, rURI("2021-09-20")) - } - case _ => super.getRelativeGraph(url) + // we introduce a circularity in the pagination of D09_07 + override def getRelativeGraph(url: RDF.URI[R]): Option[RDF.rGraph[R]] = + import scala.language.implicitConversions + url.value match + case D09_07 => + val g: RDF.rGraph[R] = + (rURI("").a(tree.Node) + -- tree.relation ->- ( + BNode().a(tree.LessThanRelation) + -- tree.node ->- rURI("2021-09-07") + -- tree.path ->- sosa.resultTime + -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) + ) + -- tree.relation ->- ( + BNode().a(tree.GreaterThanRelation) + -- tree.node ->- rURI("2021-09-09") // warning: does not exist + -- tree.path ->- sosa.resultTime + -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) + ) + -- tree.relation ->- ( + BNode().a(tree.GreaterThanRelation) + -- tree.node ->- rURI("2021-09-05") // warning: error circularity + -- tree.path ->- sosa.resultTime + -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) + )).graph ++ obsrvs(D09_07).flatten ++ ( + rURI(".").a(ldes.EventStream) + -- ldes.timestampPath ->- sosa.resultTime + -- tree.shape ->- rURI("flows-shacl") + -- tree.view ->- rURI("") + -- tree.member ->- rURI("#658") + -- tree.member ->- rURI("#3074") + -- tree.member ->- rURI("#637") + ).graph.triples.toSeq + Some(g) + case Collection => + // we add a link to a non existing view + super.getRelativeGraph(url).map { rg => + rg + rTriple(rURI(""), tree.view, rURI("2021-09-20")) + } + case _ => super.getRelativeGraph(url) diff --git a/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala b/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala index 531c522..78b805f 100644 --- a/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala +++ b/ldes/shared/src/test/scala/run/cosy/ld/ldes/MiniLdesWWW.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Typelevel + * Copyright 2021 bblfish.net * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,192 +27,188 @@ import run.cosy.ldes.prefix.{LDES, SOSA, TREE, WGS84} import scala.language.implicitConversions object MiniLdesWWW: - def mechelen(date: String): String = "https://ldes.mechelen.org/" + date - val D09_05: String = mechelen("2021-09-05") - val D09_06: String = mechelen("2021-09-06") - val D09_07: String = mechelen("2021-09-07") - val Collection: String = mechelen("") + def mechelen(date: String): String = "https://ldes.mechelen.org/" + date + val D09_05: String = mechelen("2021-09-05") + val D09_06: String = mechelen("2021-09-06") + val D09_07: String = mechelen("2021-09-07") + val Collection: String = mechelen("") class MiniLdesWWW[R <: RDF](using ops: Ops[R]) extends Web[IO, R]: - import MiniLdesWWW.* - import ops.{*, given} - val foaf = prefix.FOAF[R] - val tree = TREE[R] - val sosa = SOSA[R] - val wgs84 = WGS84[R] - val ldes = LDES[R] + import MiniLdesWWW.* + import ops.{*, given} + val foaf = prefix.FOAF[R] + val tree = TREE[R] + val sosa = SOSA[R] + val wgs84 = WGS84[R] + val ldes = LDES[R] - def area(loc: String) = rURI("area#d" + loc) - def pzDev(num: String) = URI("https://data.politie.be/sensor/dev#" + num) - def polMsr(tp: String) = URI("https://data.politie.be/sensors/measurement#" + tp) - def crop(area: String) = URI("https://data.cropland.be/area#" + area) - def cropProp(prop: String) = URI("https://data.cropland.be/measure#" + prop) + def area(loc: String) = rURI("area#d" + loc) + def pzDev(num: String) = URI("https://data.politie.be/sensor/dev#" + num) + def polMsr(tp: String) = URI("https://data.politie.be/sensors/measurement#" + tp) + def crop(area: String) = URI("https://data.cropland.be/area#" + area) + def cropProp(prop: String) = URI("https://data.cropland.be/measure#" + prop) - def get(url: RDF.URI[R]): IO[RDF.Graph[R]] = - getRelativeGraph(url) match - case Some(res) => IO(res.resolveAgainst(AbsoluteUrl.parse(url.value))) - case None => IO.raiseError[RDF.Graph[R]](new Exception(s"resource $url not reachable")) + def get(url: RDF.URI[R]): IO[RDF.Graph[R]] = getRelativeGraph(url) match + case Some(res) => IO(res.resolveAgainst(AbsoluteUrl.parse(url.value))) + case None => IO.raiseError[RDF.Graph[R]](new Exception(s"resource $url not reachable")) - def getPNG(url: RDF.URI[R]): IO[UriNGraph[R]] = - val doc = url.fragmentLess - get(doc).map(g => new UriNGraph(url, doc, g)) + def getPNG(url: RDF.URI[R]): IO[UriNGraph[R]] = + val doc = url.fragmentLess + get(doc).map(g => new UriNGraph(url, doc, g)) - def observation( - name: String, - loc: String, - simpleResult: String, - sensor: RDF.URI[R], - observedProp: RDF.URI[R], - resultTime: String - ): Seq[RDF.rTriple[R]] = - (rURI("#" + name) -- rdf.typ ->- sosa.Observation - -- wgs84.location ->- area(loc) - -- sosa.hasSimpleResult ->- (simpleResult ^^ xsd.float) - -- sosa.madeBySensor ->- sensor - -- sosa.observedProperty ->- observedProp - -- sosa.resultTime ->- (resultTime ^^ xsd.dateTimeStamp)).graph.triples.toSeq + def observation( + name: String, + loc: String, + simpleResult: String, + sensor: RDF.URI[R], + observedProp: RDF.URI[R], + resultTime: String + ): Seq[RDF.rTriple[R]] = (rURI("#" + name) -- rdf.typ ->- sosa.Observation + -- wgs84.location ->- area(loc) + -- sosa.hasSimpleResult ->- (simpleResult ^^ xsd.float) + -- sosa.madeBySensor ->- sensor + -- sosa.observedProperty ->- observedProp + -- sosa.resultTime ->- (resultTime ^^ xsd.dateTimeStamp)).graph.triples.toSeq - val obsrvs: Map[String, Seq[Seq[RDF.rTriple[R]]]] = Map( - D09_05 -> Seq( - observation( - "3", - "loc781089", - "4.0", - pzDev("213501"), - polMsr("motorized"), - "2021-09-05T23:00:00+02" - ), - observation( - "482", - "loc", - "2455.1123", - crop("schoolstraat"), - cropProp("deviceNbr"), - "2021-09-05T22:30:00+02" - ), - observation( - "4464", - "loc734383", - "10.0", - pzDev("213504+5+6"), - polMsr("bike"), - "2021-09-05T23:00:00+02" - ) - ), - D09_06 -> Seq( - observation( - "3003", - "loc763628", - "44.0", - pzDev("213503"), - polMsr("motorized"), - "2021-09-06T11:00:00+02" - ), - observation( - "4493", - "loc734383", - "197.0", - pzDev("213504+5+6"), - polMsr("motorized"), - "2021-09-06T12:00:00+02" - ), - observation( - "48", - "loc781089", - "1.0", - pzDev("213501"), - polMsr("bike"), - "2021-09-06T22:00:00+02" - ) - ), - D09_07 -> Seq( - observation( - "658", - "loc", - "5087.4795", - crop("schoolstraat"), - cropProp("deviceNbr"), - "2021-09-07T18:30:00+02" - ), - observation( - "637", - "loc", - "7009.3345", - crop("schoolstraat"), - cropProp("deviceNbr"), - "2021-09-07T13:15:00+02" - ), - observation( - "3074", - "loc763628", - "1.0", - pzDev("213503"), - polMsr("bike"), - "2021-09-06T22:00:00+02" - ) - ) - ) + val obsrvs: Map[String, Seq[Seq[RDF.rTriple[R]]]] = Map( + D09_05 -> Seq( + observation( + "3", + "loc781089", + "4.0", + pzDev("213501"), + polMsr("motorized"), + "2021-09-05T23:00:00+02" + ), + observation( + "482", + "loc", + "2455.1123", + crop("schoolstraat"), + cropProp("deviceNbr"), + "2021-09-05T22:30:00+02" + ), + observation( + "4464", + "loc734383", + "10.0", + pzDev("213504+5+6"), + polMsr("bike"), + "2021-09-05T23:00:00+02" + ) + ), + D09_06 -> Seq( + observation( + "3003", + "loc763628", + "44.0", + pzDev("213503"), + polMsr("motorized"), + "2021-09-06T11:00:00+02" + ), + observation( + "4493", + "loc734383", + "197.0", + pzDev("213504+5+6"), + polMsr("motorized"), + "2021-09-06T12:00:00+02" + ), + observation( + "48", + "loc781089", + "1.0", + pzDev("213501"), + polMsr("bike"), + "2021-09-06T22:00:00+02" + ) + ), + D09_07 -> Seq( + observation( + "658", + "loc", + "5087.4795", + crop("schoolstraat"), + cropProp("deviceNbr"), + "2021-09-07T18:30:00+02" + ), + observation( + "637", + "loc", + "7009.3345", + crop("schoolstraat"), + cropProp("deviceNbr"), + "2021-09-07T13:15:00+02" + ), + observation( + "3074", + "loc763628", + "1.0", + pzDev("213503"), + polMsr("bike"), + "2021-09-06T22:00:00+02" + ) + ) + ) - def getRelativeGraph(url: RDF.URI[R]): Option[RDF.rGraph[R]] = - relG.lift(url.value) + def getRelativeGraph(url: RDF.URI[R]): Option[RDF.rGraph[R]] = relG.lift(url.value) - def relG: PartialFunction[String, RDF.rGraph[R]] = - case Collection => - (rURI("").a(ldes.EventStream) - -- ldes.timestampPath ->- sosa.resultTime - -- tree.shape ->- rURI("flows-shacl") - -- tree.view ->- rURI("2021-09-05")).graph - case D09_05 => - (rURI("") -- rdf.typ ->- tree.Node - -- tree.relation ->- ( - BNode() -- rdf.typ ->- tree.GreaterThanRelation - -- tree.node ->- rURI("2021-09-06") - -- tree.path ->- sosa.resultTime - -- tree.value ->- ("2021-09-06T00:00:00+02" ^^ xsd.dateTimeStamp) - )).graph ++ obsrvs(D09_05).flatten ++ ( - rURI(".").a(ldes.EventStream) + def relG: PartialFunction[String, RDF.rGraph[R]] = + case Collection => (rURI("").a(ldes.EventStream) -- ldes.timestampPath ->- sosa.resultTime -- tree.shape ->- rURI("flows-shacl") - -- tree.view ->- rURI("") - -- tree.member ->- rURI("#3") - -- tree.member ->- rURI("#482") - -- tree.member ->- rURI("#4464") - ).graph.triples.toSeq - case D09_06 => - (rURI("").a(tree.Node) - -- tree.relation ->- ( - BNode().a(tree.LessThanRelation) - -- tree.node ->- rURI("2021-09-05") - -- tree.path ->- sosa.resultTime - -- tree.value ->- ("2021-09-06T00:00:00+02" ^^ xsd.dateTimeStamp) - ) - -- tree.relation ->- ( - BNode().a(tree.GreaterThanRelation) - -- tree.node ->- rURI("2021-09-07") - -- tree.path ->- sosa.resultTime - -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) - )).graph ++ obsrvs(D09_06).flatten ++ ( - rURI(".").a(ldes.EventStream) - -- ldes.timestampPath ->- sosa.resultTime - -- tree.shape ->- rURI("flows-shacl") - -- tree.view ->- rURI("") - -- tree.member ->- rURI("#4493") - -- tree.member ->- rURI("#48") - -- tree.member ->- rURI("#3003") - ).graph.triples.toSeq - case D09_07 => - (rURI("").a(tree.Node) - -- tree.relation ->- ( - BNode().a(tree.LessThanRelation) - -- tree.node ->- rURI("2021-09-06") - -- tree.path ->- sosa.resultTime - -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) - )).graph ++ obsrvs(D09_07).flatten ++ ( - rURI(".").a(ldes.EventStream) - -- ldes.timestampPath ->- sosa.resultTime - -- tree.shape ->- rURI("flows-shacl") - -- tree.view ->- rURI("") - -- tree.member ->- rURI("#658") - -- tree.member ->- rURI("#3074") - -- tree.member ->- rURI("#637") - ).graph.triples.toSeq + -- tree.view ->- rURI("2021-09-05")).graph + case D09_05 => + (rURI("") -- rdf.typ ->- tree.Node + -- tree.relation ->- ( + BNode() -- rdf.typ ->- tree.GreaterThanRelation + -- tree.node ->- rURI("2021-09-06") + -- tree.path ->- sosa.resultTime + -- tree.value ->- ("2021-09-06T00:00:00+02" ^^ xsd.dateTimeStamp) + )).graph ++ obsrvs(D09_05).flatten ++ ( + rURI(".").a(ldes.EventStream) + -- ldes.timestampPath ->- sosa.resultTime + -- tree.shape ->- rURI("flows-shacl") + -- tree.view ->- rURI("") + -- tree.member ->- rURI("#3") + -- tree.member ->- rURI("#482") + -- tree.member ->- rURI("#4464") + ).graph.triples.toSeq + case D09_06 => + (rURI("").a(tree.Node) + -- tree.relation ->- ( + BNode().a(tree.LessThanRelation) + -- tree.node ->- rURI("2021-09-05") + -- tree.path ->- sosa.resultTime + -- tree.value ->- ("2021-09-06T00:00:00+02" ^^ xsd.dateTimeStamp) + ) + -- tree.relation ->- ( + BNode().a(tree.GreaterThanRelation) + -- tree.node ->- rURI("2021-09-07") + -- tree.path ->- sosa.resultTime + -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) + )).graph ++ obsrvs(D09_06).flatten ++ ( + rURI(".").a(ldes.EventStream) + -- ldes.timestampPath ->- sosa.resultTime + -- tree.shape ->- rURI("flows-shacl") + -- tree.view ->- rURI("") + -- tree.member ->- rURI("#4493") + -- tree.member ->- rURI("#48") + -- tree.member ->- rURI("#3003") + ).graph.triples.toSeq + case D09_07 => + (rURI("").a(tree.Node) + -- tree.relation ->- ( + BNode().a(tree.LessThanRelation) + -- tree.node ->- rURI("2021-09-06") + -- tree.path ->- sosa.resultTime + -- tree.value ->- ("2021-09-07T00:00:00+02" ^^ xsd.dateTimeStamp) + )).graph ++ obsrvs(D09_07).flatten ++ ( + rURI(".").a(ldes.EventStream) + -- ldes.timestampPath ->- sosa.resultTime + -- tree.shape ->- rURI("flows-shacl") + -- tree.view ->- rURI("") + -- tree.member ->- rURI("#658") + -- tree.member ->- rURI("#3074") + -- tree.member ->- rURI("#637") + ).graph.triples.toSeq diff --git a/project/Dependencies.scala b/project/Dependencies.scala index c2515e2..28c14e5 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -1,5 +1,5 @@ -import org.portablescala.sbtplatformdeps.PlatformDepsPlugin.autoImport._ -import sbt.{Def, _} +import org.portablescala.sbtplatformdeps.PlatformDepsPlugin.autoImport.* +import sbt.{Def, *} object Dependencies { object Ver { @@ -9,24 +9,21 @@ object Dependencies { val bobcats = "0.3-3236e64-SNAPSHOT" val httpSig = "0.4-ac23f8b-SNAPSHOT" } - object other { // https://github.com/lemonlabsuk/scala-uri val scalaUri = Def.setting("io.lemonlabs" %%% "scala-uri" % "4.0.3") } - object mules { val core = Def.setting("io.chrisdavenport" %% "mules" % "0.7.0") val caffeine = Def.setting("io.chrisdavenport" %% "mules-caffeine" % "0.7.0") val http4s = Def.setting("io.chrisdavenport" %% "mules-http4s" % "0.4.0") // ember uses 0.23.18 - val ember_client = Def.setting("org.http4s" %%% "http4s-ember-client" % "0.23.18") + val ember_client = Def.setting("org.http4s" %%% "http4s-ember-client" % "0.23.18") } - // https://http4s.org/v1.0/client/ object http4s { - def apply(packg: String): Def.Initialize[sbt.ModuleID] = - Def.setting("org.http4s" %%% packg % Ver.http4s) + def apply(packg: String): Def.Initialize[sbt.ModuleID] = Def + .setting("org.http4s" %%% packg % Ver.http4s) lazy val core = http4s("http4s-core") lazy val client = http4s("http4s-client") // ember is an implementation of the client. @@ -37,7 +34,6 @@ object Dependencies { // https://search.maven.org/artifact/org.http4s/http4s-dom_sjs1_3/1.0.0-M37/jar lazy val Dom = Def.setting("org.http4s" %%% "http4s-dom" % "1.0.0-M36") } - object cats { lazy val core = Def.setting("org.typelevel" %%% "cats-core" % "2.9.0") lazy val free = Def.setting("org.typelevel" %%% "cats-free" % "2.9.0") @@ -47,10 +43,8 @@ object Dependencies { // https://github.com/typelevel/munit-cats-effect // https://search.maven.org/artifact/org.typelevel/munit-cats-effect_3/2.0.0-M3/jar - lazy val munitEffect = - Def.setting("org.typelevel" %%% "munit-cats-effect" % "2.0.0-M3") + lazy val munitEffect = Def.setting("org.typelevel" %%% "munit-cats-effect" % "2.0.0-M3") } - object crypto { // https://oss.sonatype.org/content/repositories/snapshots/net/bblfish/crypto/bobcats_3/ lazy val http4sSig = Def.setting( @@ -59,22 +53,17 @@ object Dependencies { lazy val nimbusJWT_JDK = Def.setting("com.nimbusds" % "nimbus-jose-jwt" % "9.25.4") lazy val bouncyJCA_JDK = Def.setting("org.bouncycastle" % "bcpkix-jdk18on" % "1.72") // https://oss.sonatype.org/content/repositories/snapshots/net/bblfish/crypto/bobcats_3/ - lazy val bobcats = - Def.setting("net.bblfish.crypto" %%% "bobcats" % Ver.bobcats) + lazy val bobcats = Def.setting("net.bblfish.crypto" %%% "bobcats" % Ver.bobcats) } // not published yet object banana { - lazy val bananaRdf = - Def.setting("net.bblfish.rdf" %%% "banana-rdf" % Ver.banana) + lazy val bananaRdf = Def.setting("net.bblfish.rdf" %%% "banana-rdf" % Ver.banana) lazy val bananaIO = Def.setting("net.bblfish.rdf" %%% "banana-jena-io-sync" % Ver.banana) - lazy val bananaJena = - Def.setting("net.bblfish.rdf" %%% "banana-jena-io-sync" % Ver.banana) + lazy val bananaJena = Def.setting("net.bblfish.rdf" %%% "banana-jena-io-sync" % Ver.banana) } - val scalajsDom = Def.setting("org.scala-js" %%% "scalajs-dom" % "2.0.0") - val bananaRdfLib = - Def.setting("net.bblfish.rdf" %%% "rdflibJS" % "0.9-SNAPSHOT") + val bananaRdfLib = Def.setting("net.bblfish.rdf" %%% "rdflibJS" % "0.9-SNAPSHOT") val munit = Def.setting("org.scalameta" %%% "munit" % "1.0.0-M7") // val utest = Def.setting("com.lihaoyi" %%% "utest" % "0.7.10") @@ -89,4 +78,5 @@ object Dependencies { val n3 = "n3" -> "1.11.2" val jsDom = "jsdom" -> "18.1.1" } + } diff --git a/project/build.properties b/project/build.properties index 46e43a9..72413de 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.8.2 +sbt.version=1.8.3 diff --git a/scripts/shared/src/main/scala/scripts/AnHttpSigClient.scala b/scripts/shared/src/main/scala/scripts/AnHttpSigClient.scala index 8dfb9dd..4559af2 100644 --- a/scripts/shared/src/main/scala/scripts/AnHttpSigClient.scala +++ b/scripts/shared/src/main/scala/scripts/AnHttpSigClient.scala @@ -36,9 +36,9 @@ import org.http4s.ember.client.* import run.cosy.http.headers.SigIn.KeyId object AnHttpSigClient: - implicit val runtime: IORuntime = cats.effect.unsafe.IORuntime.global + implicit val runtime: IORuntime = cats.effect.unsafe.IORuntime.global - val priv = """-----BEGIN PRIVATE KEY----- + val priv = """-----BEGIN PRIVATE KEY----- MIIEvgIBADALBgkqhkiG9w0BAQoEggSqMIIEpgIBAAKCAQEAr4tmm3r20Wd/Pbqv P1s2+QEtvpuRaV8Yq40gjUR8y2Rjxa6dpG2GXHbPfvMs8ct+Lh1GH45x28Rw3Ry5 3mm+oAXjyQ86OnDkZ5N8lYbggD4O3w6M6pAvLkhk95AndTrifbIFPNU8PPMO7Oyr @@ -67,45 +67,44 @@ object AnHttpSigClient: rOjr9w349JooGXhOxbu8nOxX -----END PRIVATE KEY-----""" - lazy val pkcs8K: PKCS8KeySpec[AsymmetricKeyAlg] = - getPrivateKeySpec(priv, AsymmetricKeyAlg.RSA_PSS_Key).get + lazy val pkcs8K: PKCS8KeySpec[AsymmetricKeyAlg] = + getPrivateKeySpec(priv, AsymmetricKeyAlg.RSA_PSS_Key).get - val keyIdStr = "http://localhost:8080/rfcKey#" - // val keyUrl: ll.Url = ll.Url("http://127.0.0.1:8080/rfcKey") - val keyUrl = URI(keyIdStr) - lazy val signerF: IO[ByteVector => IO[ByteVector]] = - Signer[IO].build(pkcs8K, bobcats.AsymmetricKeyAlg.`rsa-pss-sha512`) + val keyIdStr = "http://localhost:8080/rfcKey#" + // val keyUrl: ll.Url = ll.Url("http://127.0.0.1:8080/rfcKey") + val keyUrl = URI(keyIdStr) + lazy val signerF: IO[ByteVector => IO[ByteVector]] = Signer[IO] + .build(pkcs8K, bobcats.AsymmetricKeyAlg.`rsa-pss-sha512`) - import org.w3.banana.jena.io.JenaRDFReader.given - import org.w3.banana.jena.io.JenaRDFWriter.given + import org.w3.banana.jena.io.JenaRDFReader.given + import org.w3.banana.jena.io.JenaRDFWriter.given - lazy val keyIdData = new KeyData[IO](KeyId(Rfc8941.SfString(keyIdStr)), signerF) + lazy val keyIdData = new KeyData[IO](KeyId(Rfc8941.SfString(keyIdStr)), signerF) - given dec: RDFDecoders[IO, R] = new RDFDecoders() - import org.http4s.syntax.all.uri - given wt: WalletTools[R] = new WalletTools[R] + given dec: RDFDecoders[IO, R] = new RDFDecoders() + import org.http4s.syntax.all.uri + given wt: WalletTools[R] = new WalletTools[R] - def ioStr(uri: H4Uri): IO[String] = - emberAuthClient.flatMap(_.expect[String](uri)) + def ioStr(uri: H4Uri): IO[String] = emberAuthClient.flatMap(_.expect[String](uri)) - /** Ember Client able to authenticate with above keyId */ - def emberAuthClient: IO[Client[IO]] = - EmberClientBuilder.default[IO].build.use { (client: Client[IO]) => - import org.http4s.client.middleware.Logger - val loggedClient: Client[IO] = - Logger[IO](true, true, logAction = Some(str => IO(System.out.println(str))))(client) + /** Ember Client able to authenticate with above keyId */ + def emberAuthClient: IO[Client[IO]] = EmberClientBuilder.default[IO].build + .use { (client: Client[IO]) => + import org.http4s.client.middleware.Logger + val loggedClient: Client[IO] = + Logger[IO](true, true, logAction = Some(str => IO(System.out.println(str))))(client) - val bw = new BasicWallet[IO, R]( - Map(), - Seq(keyIdData) - )(loggedClient) + val bw = new BasicWallet[IO, R]( + Map(), + Seq(keyIdData) + )(loggedClient) - IO(AuthNClient[IO].apply(bw)(loggedClient)) - } + IO(AuthNClient[IO].apply(bw)(loggedClient)) + } - def fetch(uriStr: String = "http://localhost:8080/protected/README"): String = - // ioStr(uri"http://localhost:8080/").unsafeRunSync() - // ioStr(uri"http://localhost:8080/protected/").unsafeRunSync() - ioStr(H4Uri.unsafeFromString(uriStr)).unsafeRunSync() + def fetch(uriStr: String = "http://localhost:8080/protected/README"): String = + // ioStr(uri"http://localhost:8080/").unsafeRunSync() + // ioStr(uri"http://localhost:8080/protected/").unsafeRunSync() + ioStr(H4Uri.unsafeFromString(uriStr)).unsafeRunSync() end AnHttpSigClient diff --git a/scripts/shared/src/main/scala/scripts/MiniCF.scala b/scripts/shared/src/main/scala/scripts/MiniCF.scala index 2003350..bde7f6d 100644 --- a/scripts/shared/src/main/scala/scripts/MiniCF.scala +++ b/scripts/shared/src/main/scala/scripts/MiniCF.scala @@ -32,58 +32,49 @@ import run.cosy.ldes.LdesSpider // Mini City Flows object MiniCF: - type JR = org.w3.banana.jena.JenaRdf.type - import org.w3.banana.* - import org.w3.banana.io.{JsonLd, RDFXML, RelRDFReader, Turtle} - import org.w3.banana.jena.JenaRdf.ops - import ops.{*, given} - import org.w3.banana.jena.io.JenaRDFReader.given - given ior: unsafe.IORuntime = cats.effect.unsafe.IORuntime.global + type JR = org.w3.banana.jena.JenaRdf.type + import org.w3.banana.* + import org.w3.banana.io.{JsonLd, RDFXML, RelRDFReader, Turtle} + import org.w3.banana.jena.JenaRdf.ops + import ops.{*, given} + import org.w3.banana.jena.io.JenaRDFReader.given + given ior: unsafe.IORuntime = cats.effect.unsafe.IORuntime.global - given rdfDecoders: RDFDecoders[IO, JR] = new RDFDecoders[IO, JR] + given rdfDecoders: RDFDecoders[IO, JR] = new RDFDecoders[IO, JR] - @main - def crawlContainer(stream: String = "http://localhost:8080/ldes/miniCityFlows/stream#"): Unit = - val streamUri: RDF.URI[JR] = ops.URI(stream) + @main + def crawlContainer(stream: String = "http://localhost:8080/ldes/miniCityFlows/stream#"): Unit = + val streamUri: RDF.URI[JR] = ops.URI(stream) - val ioStr: IO[fs2.Stream[IO, Chunk[UriNGraph[JR]]]] = - AnHttpSigClient.emberAuthClient.flatMap { (client: Client[IO]) => - given web: Web[IO, JR] = new H4Web[IO, JR](client) - val spider: LdesSpider[IO, JR] = new LdesSpider[IO, JR] - spider.crawl(streamUri) - } + val ioStr: IO[fs2.Stream[IO, Chunk[UriNGraph[JR]]]] = AnHttpSigClient.emberAuthClient + .flatMap { (client: Client[IO]) => + given web: Web[IO, JR] = new H4Web[IO, JR](client) + val spider: LdesSpider[IO, JR] = new LdesSpider[IO, JR] + spider.crawl(streamUri) + } - val l: IO[List[RDF.Graph[JR]]] = - fs2.Stream - .eval(ioStr) - .flatten - .unchunks - .map(uriNG => selectObs(uriNG, streamUri)) - .unchunks - .reduce((g1, g2) => g1.union(g2)) - .compile[IO, IO, RDF.Graph[JR]] - .toList - l.unsafeRunSync().foreach(_.triples.foreach(println)) + val l: IO[List[RDF.Graph[JR]]] = fs2.Stream.eval(ioStr).flatten.unchunks + .map(uriNG => selectObs(uriNG, streamUri)).unchunks.reduce((g1, g2) => g1.union(g2)) + .compile[IO, IO, RDF.Graph[JR]].toList + l.unsafeRunSync().foreach(_.triples.foreach(println)) - end crawlContainer + end crawlContainer - import run.cosy.ldes.prefix as ldesPre - val foaf = prefix.FOAF[JR] - val tree = ldesPre.TREE[JR] - val sosa = ldesPre.SOSA[JR] - val wgs84 = ldesPre.WGS84[JR] - val ldes = ldesPre.LDES[JR] + import run.cosy.ldes.prefix as ldesPre + val foaf = prefix.FOAF[JR] + val tree = ldesPre.TREE[JR] + val sosa = ldesPre.SOSA[JR] + val wgs84 = ldesPre.WGS84[JR] + val ldes = ldesPre.LDES[JR] - /** select the observations in tree:Node of the ldes:Stream - * - * todo: it should be able to be given or fetch the shEx described in the ldes:Stream to decide - * what to select - */ - def selectObs(page: UriNGraph[JR], streamUrl: RDF.URI[JR]): fs2.Chunk[RDF.Graph[JR]] = - val collInPage = new UriNGraph[JR](streamUrl, page.name, page.graph) - val obs: Seq[RDF.Graph[JR]] = collInPage - .rel(tree.member) - .map(png => + /** select the observations in tree:Node of the ldes:Stream + * + * todo: it should be able to be given or fetch the shEx described in the ldes:Stream to decide + * what to select + */ + def selectObs(page: UriNGraph[JR], streamUrl: RDF.URI[JR]): fs2.Chunk[RDF.Graph[JR]] = + val collInPage = new UriNGraph[JR](streamUrl, page.name, page.graph) + val obs: Seq[RDF.Graph[JR]] = collInPage.rel(tree.member).map(png => png.collect( rdf.typ, wgs84.location, @@ -93,7 +84,7 @@ object MiniCF: sosa.resultTime )() ) - fs2.Chunk(obs*) - // .fold(Graph.empty)((g1, g2) => g1 union g2) + fs2.Chunk(obs*) + // .fold(Graph.empty)((g1, g2) => g1 union g2) end MiniCF From 91794b650ef8be5a1bb193bdec64d1b203bc9ea3 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Fri, 19 May 2023 20:14:59 +0200 Subject: [PATCH 25/42] initial tests for interpreted cache --- build.sbt | 2 +- .../mules/http4s/internal/CacheRules.scala | 63 +++--- .../cache/InterpretedCacheMiddleTest.scala | 114 +++++++---- .../test/scala/run/cosy/http/cache/Web.scala | 184 ++++++++++-------- 4 files changed, 214 insertions(+), 149 deletions(-) diff --git a/build.sbt b/build.sbt index 2d4dc05..53ffd70 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ import sbt.ThisBuild import sbtcrossproject.CrossPlugin.autoImport.{crossProject, CrossType} -import Dependencies.* +import Dependencies._ name := "SolidApp" ThisBuild / organization := "net.bblfish" diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/CacheRules.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/CacheRules.scala index 862a5c6..0bbf639 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/CacheRules.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/CacheRules.scala @@ -137,35 +137,40 @@ private[http4s] object CacheRules: } || response.headers.get(CIString("Pragma")).exists(_.exists(_.value === "no-cache")) def isCacheable[F[_]](req: Request[F], response: Response[F], cacheType: CacheType): Boolean = - if !cacheableMethods.contains(req.method) then - // println(s"Request Method ${req.method} - not Cacheable") - false - else if !statusIsCacheable(response.status) then - // println(s"Response Status ${response.status} - not Cacheable") - false - else if cacheControlNoStoreExists(response) then - // println("Cache-Control No-Store is present - not Cacheable") - false - else if cacheType.isShared && cacheControlPrivateExists(response) then - // println("Cache is shared and Cache-Control private exists - not Cacheable") - false - else if cacheType.isShared && response.headers.get(CIString("Vary")) - .exists(h => h.exists(_.value === "*")) - then - // println("Cache is shared and Vary header exists as * - not Cacheable") - false - else if cacheType.isShared && authorizationHeaderExists(response) && !cacheControlPublicExists( - response - ) - then - // println("Cache is Shared and Authorization Header is present and Cache-Control public is not present - not Cacheable") - false - else if mustRevalidate(response) && !(response.headers.get[ETag].isDefined || response.headers - .get[`Last-Modified`].isDefined) - then false - else if req.method === Method.GET || req.method === Method.HEAD then true - else if cacheControlPublicExists(response) || cacheControlPrivateExists(response) then true - else response.headers.get[Expires].isDefined + val cachable = + if !cacheableMethods.contains(req.method) then + // println(s"Request Method ${req.method} - not Cacheable") + false + else if !statusIsCacheable(response.status) then + // println(s"Response Status ${response.status} - not Cacheable") + false + else if cacheControlNoStoreExists(response) then + // println("Cache-Control No-Store is present - not Cacheable") + false + else if cacheType.isShared && cacheControlPrivateExists(response) then + // println("Cache is shared and Cache-Control private exists - not Cacheable") + false + else if cacheType.isShared && response.headers.get(CIString("Vary")) + .exists(h => h.exists(_.value === "*")) + then + // println("Cache is shared and Vary header exists as * - not Cacheable") + false + else if cacheType.isShared && authorizationHeaderExists( + response + ) && !cacheControlPublicExists( + response + ) + then + // println("Cache is Shared and Authorization Header is present and Cache-Control public is not present - not Cacheable") + false + else if mustRevalidate(response) && !(response.headers.get[ETag].isDefined || response + .headers.get[`Last-Modified`].isDefined) + then false + else if req.method === Method.GET || req.method === Method.HEAD then true + else if cacheControlPublicExists(response) || cacheControlPrivateExists(response) then true + else response.headers.get[Expires].isDefined + // println(s"Cacheable($req, $response)= " + cachable) + cachable def shouldInvalidate[F[_]](request: Request[F], response: Response[F]): Boolean = if Set(Status.NotFound, Status.Gone).contains(response.status) then true diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala b/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala index 81bf653..9379fc1 100644 --- a/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala +++ b/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala @@ -1,19 +1,3 @@ -/* - * Copyright 2021 bblfish.net - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package run.cosy.http.cache import cats.effect.{IO, Async} @@ -50,7 +34,7 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: Web.httpRoutes[IO].orNotFound.run(Request[IO](uri = Uri(path = Root))).map { response => assertEquals(response.status, Status.Ok) assertEquals( - response.headers, + removeDate(response.headers), Web.headers("/") ) } >> Web.httpRoutes[IO].orNotFound.run( @@ -60,7 +44,7 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: ).map { response => assertEquals(response.status, Status.Ok) assertEquals( - response.headers, + removeDate(response.headers), Web.headers("world-at-peace", Some("/people/henry/blog/"), MediaType.text.plain) ) assertEquals( @@ -70,11 +54,23 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: } } def bytesToString(bytes: Vector[Byte]): String = bytes.map(_.toChar).mkString + def parseMap(body: String): Map[String, Int] = body.split("\n").map(_.split(" -> ")) + .map { case Array(path, count) => (path, count.toInt) }.toMap - test("test string cache") { - for ref <- Ref.of[IO, WebCache[CacheItem[String]]](Map.empty) - yield - val clientMiddleWare = InterpretedCacheMiddleware.app[IO, String]( + val worldPeace: Uri = Uri + .unsafeFromString("https://bblfish.net/people/henry/blog/2023/04/01/world-at-peace") + val bbl: Uri = Uri.unsafeFromString("https://bblfish.net/") + val counterUri = Uri.unsafeFromString("https://bblfish.net/counter") + + /** we need dates so that caching can work but it is tricky to compare them */ + def removeDate(headers: Headers): Headers = + import org.typelevel.ci.CIStringSyntax + headers.transform(l => l.filterNot(_.name == ci"Date")) + + test("test String Cache") { + for + ref <- Ref.of[IO, WebCache[CacheItem[String]]](Map.empty) + stringCacheMiddleWare = InterpretedCacheMiddleware.app[IO, String]( TreeDirCache[IO, CacheItem[String]](ref), (response: Response[IO]) => response.bodyText.compile.toVector.map { vec => @@ -86,32 +82,66 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: ) } ) - - val bbl: Uri = Uri - .unsafeFromString("https://bblfish.net/people/henry/blog/2023/04/01/world-at-peace") - assertNotEquals(bbl, null) - val interpretedClient = clientMiddleWare(Web.httpRoutes[IO].orNotFound) - val req = Request[IO](Method.GET, bbl) - val resp: CachedResponse[String] = interpretedClient.run(req).unsafeRunSync() - assertEquals(resp.status, Status.Ok) + cachedClient = stringCacheMiddleWare(Web.httpRoutes[IO].orNotFound) + respWP1 <- cachedClient.run(Request[IO](Method.GET, worldPeace)) + respRoot <- cachedClient.run(Request[IO](Method.GET, bbl)) + rescounter1 <- cachedClient.run(Request[IO](Method.GET, counterUri)) + respWP2 <- cachedClient.run(Request[IO](Method.GET, worldPeace)) + rescounter2 <- cachedClient.run(Request[IO](Method.GET, counterUri)) + respRoot2 <- cachedClient.run(Request[IO](Method.GET, bbl)) + rescounter3 <- cachedClient.run(Request[IO](Method.GET, counterUri)) + yield + assertEquals(respWP1.status, Status.Ok) assertEquals( - resp.headers, + removeDate(respWP1.headers), Web.headers("world-at-peace", Some("/people/henry/blog/"), MediaType.text.plain) ) assertEquals( - resp.body, + respWP1.body, Some("Hello World!") ) - // val resp2 = client(Web.httpRoutes[IO].orNotFound).run(req).unsafeRunSync() - // assertEquals(resp2.status, Status.Ok) - // assertEquals( - // resp2.headers, - // Web.headers("/") - // ) - // assertEquals( - // resp2.body.compile.toVector.unsafeRunSync(), - // ByteVector("Hello World!".getBytes()) - // ) + assertEquals(respRoot.status, Status.Ok) + assertEquals( + removeDate(respRoot.headers), + Web.headers("/") + ) + assertEquals( + respRoot.body, + Some(Web.rootTtl) + ) + assertEquals(rescounter1.status, Status.Ok) + rescounter1.body.map { body => + val c1 = parseMap(body) + assertEquals(c1(worldPeace.path.toString), 1) + assertEquals(c1(bbl.path.toString), 1) + assertEquals(c1(counterUri.path.toString), 1) + }.getOrElse(fail("no body")) + + assertEquals(respWP2.status, Status.Ok) + assertEquals( + removeDate(respWP1.headers), + Web.headers("world-at-peace", Some("/people/henry/blog/"), MediaType.text.plain) + ) + assertEquals(rescounter2.status, Status.Ok) + rescounter2.body.map { body => + val c1 = parseMap(body) + assertEquals(c1(worldPeace.path.toString), 1) + assertEquals(c1(bbl.path.toString), 1) + assertEquals(c1(counterUri.path.toString), 1) + }.getOrElse(fail("no body for 2nd req to " + counterUri)) + + assertEquals(respRoot2.status, Status.Ok) + assertEquals( + respRoot2.body, + Some(Web.rootTtl) + ) + rescounter3.body.map { body => + val c1 = parseMap(body) + assertEquals(c1(worldPeace.path.toString), 1) + assertEquals(c1(bbl.path.toString), 1) + assertEquals(c1(counterUri.path.toString), 1) + }.getOrElse(fail("no body for 3rd req to " + counterUri)) + } end InterpretedCacheMiddleTest diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala b/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala index ea2a0a3..ceb57ca 100644 --- a/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala +++ b/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala @@ -25,10 +25,16 @@ import org.http4s.Uri.Path.Root import org.http4s.Method.{GET, HEAD} import scodec.bits.ByteVector import org.http4s.dsl.io.* +import org.http4s.CharsetRange.Atom +import java.util.concurrent.atomic.AtomicReference +import cats.data.Kleisli +import cats.FlatMap +import cats.data.OptionT +import cats.Monad +import cats.effect.kernel.Clock // import io.chrisdavenport.mules.http4s.CachedResponse.body object Web: - // implicit def toEnt[F](str: String): Entity[F] = extension (uri: Uri) /** Uri("/") + ".acl" == Uri("/.acl") and Uri("foo")+".acl" == Uri("foo.acl") */ @@ -59,86 +65,110 @@ object Web: // relative URLs val thisDoc = Uri.unsafeFromString("") val thisDir = Uri.unsafeFromString(".") + val rootAcr = """ + |@prefix wac: . + |@prefix foaf: . + | + |<#R1> a wac:Authorization; + | wac:mode wac:Control; + | wac:agent ; + | wac:default <.> . + | + |<#R2> a wac:Authorization; + | wac:mode wac:Read; + | wac:agentClass foaf:Agent; + | wac:accessTo <.> ; + | wac:default <.> . + """.stripMargin - def httpRoutes[F[_]](using - AS: Async[F] - ): HttpRoutes[F] = HttpRoutes.of[F] { - case GET -> Root => AS.pure( - Response[F]( - status = Status.Ok, - entity = Entity.strict(ByteVector(""" - |@prefix ldp: . - | - |<> a ldp:BasicContainer; - | ldp:contains . - |""".stripMargin.getBytes())), - headers = headers("/") - ) - ) - - case GET -> Root / ".acr" => AS.pure( - Response[F]( - status = Status.Ok, - entity = Entity.strict(ByteVector(""" - |@prefix wac: . - |@prefix foaf: . - | - |<#R1> a wac:Authorization; - | wac:mode wac:Control; - | wac:agent ; - | wac:default <.> . - | - |<#R2> a wac:Authorization; - | wac:mode wac:Read; - | wac:agentClass foaf:Agent; - | wac:accessTo <.> ; - | wac:default <.> . - |""".stripMargin.getBytes())), - headers = headers("/") - ) - ) + val rootTtl = """ + |@prefix ldp: . + | + |<> a ldp:BasicContainer; + | ldp:contains . + |""".stripMargin - case GET -> Root / "people" / "henry" / "card" => AS.pure( - Response( - status = Status.Ok, - entity = Entity.strict(ByteVector(""" - |<#i> a foaf:Person; - | foaf.name "Henry Story"; - | foaf.knows . - """.stripMargin.getBytes())), - headers = headers("card", Some("/")) - ) - ) + val bblCardTtl = """ + |<#i> a foaf:Person; + | + | foaf.name "Henry Story"; + | foaf.knows . + """.stripMargin - case GET -> Root / "people" / "henry" / "blog" / "2023" / "04" / "01" / "world-at-peace" => - OK[F]( - "Hello World!", - headers("world-at-peace", Some("/people/henry/blog/"), MediaType.text.plain) - ) + val bblBlogAcr = """ + @prefix wac: . + @prefix foaf: . + + <#R1> a wac:Authorization; + wac:mode wac:Control; + wac:agent ; + wac:default <.> . + + <#R2> a wac:Authorization; + wac:mode wac:Read; + wac:agentClass foaf:Agent; + wac:default <.> . + + <#R3> a wac:Authorization; + wac:mode wac:Read; + wac:agentClass foaf:Agent; + wac:default <.> . + """.stripMargin - case GET -> Root / "people" / "henry" / "blog" / ".acr" => OK[F]( - """ - |@prefix wac: . - |@prefix foaf: . - | - |<#R1> a wac:Authorization; - | wac:mode wac:Control; - | wac:agent ; - | wac:default <.> . - | - |<#R2> a wac:Authorization; - | wac:mode wac:Read; - | wac:agentClass foaf:Agent; - | wac:default <.> . - | - |<#R3> a wac:Authorization; - | wac:mode wac:Read; - | wac:agentClass foaf:Agent; - | wac:default <.> . - """.stripMargin, - headers("") - ) - } + def httpRoutes[F[_]: Monad: Clock](using + AS: Async[F] + ): HttpRoutes[F] = + val counter = AtomicReference(Map.empty[Uri.Path, Int]) + val inc = Kleisli[OptionT[F, *], Request[F], Request[F]] { req => + OptionT.liftF(AS.delay { + counter.updateAndGet { m => + val count = m.getOrElse(req.uri.path, 0) + m.updated(req.uri.path, count + 1) + } + }) >> OptionT.pure(req) + } + val routes: Kleisli[OptionT[F, *], Request[F], Response[F]] = HttpRoutes.of[F] { + case GET -> Root => AS.pure( + Response[F]( + status = Status.Ok, + entity = Entity.strict(ByteVector(rootTtl.getBytes())), + headers = headers("/") + ) + ) + case GET -> Root / ".acr" => AS.pure( + Response[F]( + status = Status.Ok, + entity = Entity.strict(ByteVector(rootAcr.getBytes())), + headers = headers("/") + ) + ) + case GET -> Root / "people" / "henry" / "card" => AS.pure( + Response( + status = Status.Ok, + entity = Entity.strict(ByteVector(bblCardTtl.getBytes())), + headers = headers("card", Some("/")) + ) + ) + case GET -> Root / "people" / "henry" / "blog" / "2023" / "04" / "01" / "world-at-peace" => + OK[F]( + "Hello World!", + headers("world-at-peace", Some("/people/henry/blog/"), MediaType.text.plain) + ) + case GET -> Root / "people" / "henry" / "blog" / ".acr" => OK[F]( + bblBlogAcr, + headers("") + ) + case GET -> Root / "counter" => + val cntStr = counter.get().toList.map { (p, i) => p.toString + " -> " + i }.mkString("\n") + OK[F](cntStr, headers("/counter", Some("/"), MediaType.text.plain)) + } + val addTime: Kleisli[OptionT[F, *], Response[F], Response[F]] = + Kleisli[OptionT[F, *], Response[F], Response[F]] { resp => + OptionT.liftF[F,HttpDate](HttpDate.current[F]).map { now => + resp.putHeaders(Date(now)) + } + } + inc andThen routes andThen addTime def OK[F[_]: Async](body: String, headers: Headers): F[Response[F]] = Async[F].pure( Response[F]( From 2954d6893edab539e39ee675fa9f34719de5f616 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Fri, 19 May 2023 20:29:33 +0200 Subject: [PATCH 26/42] add Chris Davenport's license to his code --- .../mules/http4s/CacheItem.scala | 2 +- .../mules/http4s/CacheType.scala | 21 ++++-------- .../mules/http4s/CachedResponse.scala | 3 +- .../mules/http4s/internal/CacheRules.scala | 32 ++++++++++--------- 4 files changed, 26 insertions(+), 32 deletions(-) diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala index 602deb6..b13ca62 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 bblfish.net + * * Copyright 2019 Chris Davenport * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheType.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheType.scala index 53b9b34..aff7503 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheType.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheType.scala @@ -1,18 +1,11 @@ /* - * Copyright 2021 bblfish.net - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + * Copyright 2019 Christopher Davenport + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ package io.chrisdavenport.mules.http4s diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala index d1ce5ab..5a278ef 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala @@ -1,6 +1,5 @@ /* - * Copyright 2021 bblfish.net - * + * Copyright 2019 Chris Davenport * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/CacheRules.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/CacheRules.scala index 0bbf639..d624b5d 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/CacheRules.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/CacheRules.scala @@ -1,18 +1,20 @@ -/* - * Copyright 2021 bblfish.net - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +/** Copyright 2019 Christopher Davenport + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software + * and associated documentation files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING + * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ package io.chrisdavenport.mules.http4s.internal From 013acc6b6ee0f086279205cd2f387ded40d40550 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Fri, 19 May 2023 20:34:32 +0200 Subject: [PATCH 27/42] add margins to Web strings --- .../test/scala/run/cosy/http/cache/Web.scala | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala b/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala index ceb57ca..e0483fb 100644 --- a/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala +++ b/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala @@ -79,7 +79,7 @@ object Web: | wac:agentClass foaf:Agent; | wac:accessTo <.> ; | wac:default <.> . - """.stripMargin + |""".stripMargin val rootTtl = """ |@prefix ldp: . @@ -93,33 +93,34 @@ object Web: | | foaf.name "Henry Story"; | foaf.knows . - """.stripMargin + |""".stripMargin val bblBlogAcr = """ - @prefix wac: . - @prefix foaf: . - - <#R1> a wac:Authorization; - wac:mode wac:Control; - wac:agent ; - wac:default <.> . - - <#R2> a wac:Authorization; - wac:mode wac:Read; - wac:agentClass foaf:Agent; - wac:default <.> . - - <#R3> a wac:Authorization; - wac:mode wac:Read; - wac:agentClass foaf:Agent; - wac:default <.> . - """.stripMargin + |@prefix wac: . + |@prefix foaf: . + | + |<#R1> a wac:Authorization; + | wac:mode wac:Control; + | wac:agent ; + | wac:default <.> . + | + |<#R2> a wac:Authorization; + | wac:mode wac:Read; + | wac:agentClass foaf:Agent; + | wac:default <.> . + | + |<#R3> a wac:Authorization; + | wac:mode wac:Read; + | wac:agentClass foaf:Agent; + | wac:default <.> . + |""".stripMargin def httpRoutes[F[_]: Monad: Clock](using AS: Async[F] ): HttpRoutes[F] = val counter = AtomicReference(Map.empty[Uri.Path, Int]) val inc = Kleisli[OptionT[F, *], Request[F], Request[F]] { req => + ntln("server received request " + req.method + req.uri.path) OptionT.liftF(AS.delay { counter.updateAndGet { m => val count = m.getOrElse(req.uri.path, 0) @@ -163,11 +164,11 @@ object Web: OK[F](cntStr, headers("/counter", Some("/"), MediaType.text.plain)) } val addTime: Kleisli[OptionT[F, *], Response[F], Response[F]] = - Kleisli[OptionT[F, *], Response[F], Response[F]] { resp => - OptionT.liftF[F,HttpDate](HttpDate.current[F]).map { now => + Kleisli[OptionT[F, *], Response[F], Response[F]] { resp => + OptionT.liftF[F, HttpDate](HttpDate.current[F]).map { now => resp.putHeaders(Date(now)) - } - } + } + } inc andThen routes andThen addTime def OK[F[_]: Async](body: String, headers: Headers): F[Response[F]] = Async[F].pure( From 116a26059825e2ee7c64f4de6e54cfdbb255d47a Mon Sep 17 00:00:00 2001 From: Henry Story Date: Sat, 20 May 2023 10:05:20 +0200 Subject: [PATCH 28/42] remove println --- cache/shared/src/test/scala/run/cosy/http/cache/Web.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala b/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala index e0483fb..adc8c3f 100644 --- a/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala +++ b/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala @@ -79,7 +79,7 @@ object Web: | wac:agentClass foaf:Agent; | wac:accessTo <.> ; | wac:default <.> . - |""".stripMargin + | """.stripMargin val rootTtl = """ |@prefix ldp: . @@ -120,7 +120,6 @@ object Web: ): HttpRoutes[F] = val counter = AtomicReference(Map.empty[Uri.Path, Int]) val inc = Kleisli[OptionT[F, *], Request[F], Request[F]] { req => - ntln("server received request " + req.method + req.uri.path) OptionT.liftF(AS.delay { counter.updateAndGet { m => val count = m.getOrElse(req.uri.path, 0) From 672c554a1d2370fddd08b5ab8552fb74973c8e8d Mon Sep 17 00:00:00 2001 From: Henry Story Date: Sat, 20 May 2023 10:43:10 +0200 Subject: [PATCH 29/42] reformat some copyright info --- .../mules/http4s/CacheItem.scala | 32 ++++++++++--------- .../mules/http4s/CacheType.scala | 25 ++++++++++----- .../mules/http4s/CachedResponse.scala | 31 ++++++++++-------- 3 files changed, 51 insertions(+), 37 deletions(-) diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala index b13ca62..15a561e 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala @@ -1,18 +1,20 @@ -/* - * * Copyright 2019 Chris Davenport - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +/** Copyright 2019 Christopher Davenport + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software + * and associated documentation files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING + * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ package io.chrisdavenport.mules.http4s diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheType.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheType.scala index aff7503..75b3b0d 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheType.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheType.scala @@ -1,11 +1,20 @@ -/* - * Copyright 2019 Christopher Davenport - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/** Copyright 2019 Christopher Davenport + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software + * and associated documentation files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING + * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ package io.chrisdavenport.mules.http4s diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala index 5a278ef..181daa2 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala @@ -1,17 +1,20 @@ -/* - * Copyright 2019 Chris Davenport - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +/** Copyright 2019 Christopher Davenport + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software + * and associated documentation files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING + * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ package io.chrisdavenport.mules.http4s From e82f763ff978a931d09ba1dd750b9fdb8b927a99 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Sat, 20 May 2023 18:57:43 +0200 Subject: [PATCH 30/42] tests for getting & finding the cloest dir info --- .../mules/http4s/internal/Caching.scala | 32 ++++++++------- .../scala/run/cosy/http/cache/Cache.scala | 5 +++ .../scala/run/cosy/http/cache/DirTree.scala | 6 ++- .../cache/InterpretedCacheMiddleTest.scala | 39 +++++++++++++++++-- .../test/scala/run/cosy/http/cache/Web.scala | 28 +++++++++++-- 5 files changed, 86 insertions(+), 24 deletions(-) diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala index e314f66..b9bc707 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala @@ -1,18 +1,20 @@ -/* - * Copyright 2021 bblfish.net - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +/** Copyright 2019 Christopher Davenport + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software + * and associated documentation files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING + * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ package io.chrisdavenport.mules.http4s.internal diff --git a/cache/shared/src/main/scala/run/cosy/http/cache/Cache.scala b/cache/shared/src/main/scala/run/cosy/http/cache/Cache.scala index a34b951..63f9cf3 100644 --- a/cache/shared/src/main/scala/run/cosy/http/cache/Cache.scala +++ b/cache/shared/src/main/scala/run/cosy/http/cache/Cache.scala @@ -94,6 +94,11 @@ case class TreeDirCache[F[_], X]( val (path, v) = tree.find(k.path.segments) if path.isEmpty then v else None + /** find the closest node matching `select` going backwards from the closest node we have leading + * to path. So if we want but we have + * and but only that content at the latter + * resource matches, then we will get that. + */ def findClosest(k: Uri)(matcher: Option[X] => Boolean): F[Option[X]] = for scheme <- F.fromOption(k.scheme, IncompleteServiceInfo(k)) diff --git a/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala b/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala index 0758082..b5279e0 100644 --- a/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala +++ b/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala @@ -86,7 +86,11 @@ object DirTree: loop(thizDt, path, Seq()) end unzipAlong - /** find the closest node matching `select` going backwards from where we got */ + /** find the closest node matching `select` going backwards from the closest node we have + * leading to path. So if we want + * but we have and but only that content + * at the latter resource matches, then we will get that. + */ def findClosest(path: Path)(select: X => Boolean): Option[X] = unzipAlong(path) match case (Right(dt), zpath) => dt.head +: zpath.map(_.from.head) find select case (Left(_), zpath) => zpath.map(_.from.head) find select diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala b/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala index 9379fc1..22e11e1 100644 --- a/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala +++ b/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala @@ -14,6 +14,7 @@ import cats.effect.kernel.Ref import cats.data.Kleisli import io.chrisdavenport.mules.http4s.CachedResponse import io.chrisdavenport.mules.http4s.CacheItem +import org.typelevel.ci.CIStringSyntax object Test: // Return true if match succeeds; otherwise false @@ -59,19 +60,20 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: val worldPeace: Uri = Uri .unsafeFromString("https://bblfish.net/people/henry/blog/2023/04/01/world-at-peace") + val bblBlogVirgin: Uri = Uri.unsafeFromString("https://bblfish.net/people/henry/blog/2023/05/18/birth") val bbl: Uri = Uri.unsafeFromString("https://bblfish.net/") val counterUri = Uri.unsafeFromString("https://bblfish.net/counter") /** we need dates so that caching can work but it is tricky to compare them */ - def removeDate(headers: Headers): Headers = - import org.typelevel.ci.CIStringSyntax - headers.transform(l => l.filterNot(_.name == ci"Date")) + def removeDate(headers: Headers): Headers = headers + .transform(l => l.filterNot(_.name == ci"Date")) test("test String Cache") { for ref <- Ref.of[IO, WebCache[CacheItem[String]]](Map.empty) + cache = TreeDirCache[IO, CacheItem[String]](ref) stringCacheMiddleWare = InterpretedCacheMiddleware.app[IO, String]( - TreeDirCache[IO, CacheItem[String]](ref), + cache, (response: Response[IO]) => response.bodyText.compile.toVector.map { vec => CachedResponse( @@ -90,6 +92,21 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: rescounter2 <- cachedClient.run(Request[IO](Method.GET, counterUri)) respRoot2 <- cachedClient.run(Request[IO](Method.GET, bbl)) rescounter3 <- cachedClient.run(Request[IO](Method.GET, counterUri)) + cachedWorldPeace <- cache.lookup(worldPeace) + closestDir1 <- cache.findClosest(bblBlogVirgin)(_.isDefined) + wpLink = cachedWorldPeace.flatMap { ci => + val x: List[Uri] = for + links <- ci.response.headers.get[Link].toList + link <- links.values.toList + if link.rel.contains(Web.defaultAccessContainer) + yield worldPeace.resolve(link.uri) + x.headOption //there should be only one! + } + blogAcrDirUrl <- IO.fromOption(wpLink)( + new Exception(s"no default container Link header in $cachedWorldPeace") + ) + cacheBlogDir <- cachedClient.run(Request[IO](Method.GET, blogAcrDirUrl)) + closestDir2 <- cache.findClosest(bblBlogVirgin)(_.isDefined) yield assertEquals(respWP1.status, Status.Ok) assertEquals( @@ -110,6 +127,7 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: Some(Web.rootTtl) ) assertEquals(rescounter1.status, Status.Ok) + //this tells us that we hit the cache once rescounter1.body.map { body => val c1 = parseMap(body) assertEquals(c1(worldPeace.path.toString), 1) @@ -123,6 +141,8 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: Web.headers("world-at-peace", Some("/people/henry/blog/"), MediaType.text.plain) ) assertEquals(rescounter2.status, Status.Ok) + + //this tells us that we got all the info directly from the cache rescounter2.body.map { body => val c1 = parseMap(body) assertEquals(c1(worldPeace.path.toString), 1) @@ -142,6 +162,17 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: assertEquals(c1(counterUri.path.toString), 1) }.getOrElse(fail("no body for 3rd req to " + counterUri)) + // test direct access to cache + assertEquals(cachedWorldPeace.get.response.status, Status.Ok) + assertEquals(cachedWorldPeace.get.response.body, Some(Web.bblWorldAtPeace)) + + assertEquals(cacheBlogDir.status, Status.Ok) + assertEquals( + removeDate(cacheBlogDir.headers), + Web.headers("") + ) + assertEquals(closestDir1.map(_.response.body), Some(Some(Web.rootTtl))) + assertEquals(closestDir2.map(_.response.body), Some(Some(Web.bblBlogRootContainer))) } end InterpretedCacheMiddleTest diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala b/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala index adc8c3f..c6e8906 100644 --- a/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala +++ b/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala @@ -56,11 +56,12 @@ object Web: `Content-Type`(mime), Link( LinkValue(docAcrUri, rel = Some("acl")), - defaultUri.map(ac => LinkValue(ac, rel = Some("defaultAccessContainer"))).toSeq* + defaultUri.map(ac => LinkValue(ac, rel = Some(defaultAccessContainer))).toSeq* ), Allow(GET, HEAD) ) - + + val defaultAccessContainer = "defaultAccessContainer" val rootDir = Uri(path = Root) // relative URLs val thisDoc = Uri.unsafeFromString("") @@ -115,6 +116,15 @@ object Web: | wac:default <.> . |""".stripMargin + val bblBlogRootContainer = """ + |@prefix ldp: . + |<> a ldp:BasicContainer; + | . ldp:contains <2023/> . + |""".stripMargin + + val bblWorldAtPeace = "Hello World!" + val bblBlogVirgin = "Play in three acts" + def httpRoutes[F[_]: Monad: Clock](using AS: Async[F] ): HttpRoutes[F] = @@ -149,9 +159,19 @@ object Web: headers = headers("card", Some("/")) ) ) - case GET -> Root / "people" / "henry" / "blog" / "2023" / "04" / "01" / "world-at-peace" => + case GET -> Root / "people" / "henry" / "blog" / "2023" / "05" / "18" / "birth" => + OK[F]( + bblBlogVirgin, + headers("birth", Some("/people/henry/blog/"), MediaType.text.plain) + ) + case GET -> Root / "people" / "henry" / "blog" / "" => // <- is this "ends with slash"? + OK[F]( + bblBlogRootContainer, + headers("") + ) + case GET -> Root / "people" / "henry" / "blog" / "2023" / "04" / "01" / "world-at-peace" => OK[F]( - "Hello World!", + bblWorldAtPeace, headers("world-at-peace", Some("/people/henry/blog/"), MediaType.text.plain) ) case GET -> Root / "people" / "henry" / "blog" / ".acr" => OK[F]( From c94d63aadbb1d9bc2655b65cf7fc95026bf8e927 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Sat, 20 May 2023 19:13:46 +0200 Subject: [PATCH 31/42] minor refactoring --- .../cache/InterpretedCacheMiddleTest.scala | 73 +++++++++---------- 1 file changed, 33 insertions(+), 40 deletions(-) diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala b/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala index 22e11e1..49c2fbc 100644 --- a/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala +++ b/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala @@ -16,21 +16,21 @@ import io.chrisdavenport.mules.http4s.CachedResponse import io.chrisdavenport.mules.http4s.CacheItem import org.typelevel.ci.CIStringSyntax -object Test: -// Return true if match succeeds; otherwise false -// def check[A](actual: IO[Response[IO]], expectedStatus: Status, expectedBody: Option[A])(using -// ev: EntityDecoder[IO, A] -// ): Boolean = -// val actualResp = actual.unsafeRunSync() -// val statusCheck = actualResp.status == expectedStatus -// val bodyCheck = expectedBody.fold[Boolean]( -// // Verify Response's body is empty. -// actualResp.body.compile.toVector.unsafeRunSync().isEmpty -// )(expected => actualResp.as[A].unsafeRunSync() == expected) -// statusCheck && bodyCheck -end Test +object InterpretedCacheMiddleTest: + def bytesToString(bytes: Vector[Byte]): String = bytes.map(_.toChar).mkString + def parseMap(body: String): Map[String, Int] = body.split("\n").map(_.split(" -> ")) + .map { case Array(path, count) => (path, count.toInt) }.toMap + + val worldPeace: Uri = Uri + .unsafeFromString("https://bblfish.net/people/henry/blog/2023/04/01/world-at-peace") + val bblBlogVirgin: Uri = Uri + .unsafeFromString("https://bblfish.net/people/henry/blog/2023/05/18/birth") + val bbl: Uri = Uri.unsafeFromString("https://bblfish.net/") + val counterUri = Uri.unsafeFromString("https://bblfish.net/counter") +end InterpretedCacheMiddleTest class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: + import InterpretedCacheMiddleTest.* test("test web") { Web.httpRoutes[IO].orNotFound.run(Request[IO](uri = Uri(path = Root))).map { response => assertEquals(response.status, Status.Ok) @@ -54,16 +54,6 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: ) } } - def bytesToString(bytes: Vector[Byte]): String = bytes.map(_.toChar).mkString - def parseMap(body: String): Map[String, Int] = body.split("\n").map(_.split(" -> ")) - .map { case Array(path, count) => (path, count.toInt) }.toMap - - val worldPeace: Uri = Uri - .unsafeFromString("https://bblfish.net/people/henry/blog/2023/04/01/world-at-peace") - val bblBlogVirgin: Uri = Uri.unsafeFromString("https://bblfish.net/people/henry/blog/2023/05/18/birth") - val bbl: Uri = Uri.unsafeFromString("https://bblfish.net/") - val counterUri = Uri.unsafeFromString("https://bblfish.net/counter") - /** we need dates so that caching can work but it is tricky to compare them */ def removeDate(headers: Headers): Headers = headers .transform(l => l.filterNot(_.name == ci"Date")) @@ -84,28 +74,29 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: ) } ) - cachedClient = stringCacheMiddleWare(Web.httpRoutes[IO].orNotFound) - respWP1 <- cachedClient.run(Request[IO](Method.GET, worldPeace)) - respRoot <- cachedClient.run(Request[IO](Method.GET, bbl)) - rescounter1 <- cachedClient.run(Request[IO](Method.GET, counterUri)) - respWP2 <- cachedClient.run(Request[IO](Method.GET, worldPeace)) - rescounter2 <- cachedClient.run(Request[IO](Method.GET, counterUri)) - respRoot2 <- cachedClient.run(Request[IO](Method.GET, bbl)) - rescounter3 <- cachedClient.run(Request[IO](Method.GET, counterUri)) + cc = stringCacheMiddleWare(Web.httpRoutes[IO].orNotFound) + respWP1 <- cc.run(Request[IO](GET, worldPeace)) + respRoot <- cc.run(Request[IO](GET, bbl)) + rescounter1 <- cc.run(Request[IO](GET, counterUri)) + respWP2 <- cc.run(Request[IO](GET, worldPeace)) + rescounter2 <- cc.run(Request[IO](GET, counterUri)) + respRoot2 <- cc.run(Request[IO](GET, bbl)) + rescounter3 <- cc.run(Request[IO](GET, counterUri)) cachedWorldPeace <- cache.lookup(worldPeace) closestDir1 <- cache.findClosest(bblBlogVirgin)(_.isDefined) wpLink = cachedWorldPeace.flatMap { ci => - val x: List[Uri] = for - links <- ci.response.headers.get[Link].toList - link <- links.values.toList - if link.rel.contains(Web.defaultAccessContainer) - yield worldPeace.resolve(link.uri) - x.headOption //there should be only one! + val x: List[Uri] = + for + links <- ci.response.headers.get[Link].toList + link <- links.values.toList + if link.rel.contains(Web.defaultAccessContainer) + yield worldPeace.resolve(link.uri) + x.headOption // there should be only one! } blogAcrDirUrl <- IO.fromOption(wpLink)( new Exception(s"no default container Link header in $cachedWorldPeace") ) - cacheBlogDir <- cachedClient.run(Request[IO](Method.GET, blogAcrDirUrl)) + cacheBlogDir <- cc.run(Request[IO](Method.GET, blogAcrDirUrl)) closestDir2 <- cache.findClosest(bblBlogVirgin)(_.isDefined) yield assertEquals(respWP1.status, Status.Ok) @@ -127,7 +118,7 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: Some(Web.rootTtl) ) assertEquals(rescounter1.status, Status.Ok) - //this tells us that we hit the cache once + // this tells us that we hit the cache once rescounter1.body.map { body => val c1 = parseMap(body) assertEquals(c1(worldPeace.path.toString), 1) @@ -142,7 +133,7 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: ) assertEquals(rescounter2.status, Status.Ok) - //this tells us that we got all the info directly from the cache + // this tells us that we got all the info directly from the cache rescounter2.body.map { body => val c1 = parseMap(body) assertEquals(c1(worldPeace.path.toString), 1) @@ -171,7 +162,9 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: removeDate(cacheBlogDir.headers), Web.headers("") ) + // the closest dir is the root because we have not cached the blog dir assertEquals(closestDir1.map(_.response.body), Some(Some(Web.rootTtl))) + // now the blog dir is cached assertEquals(closestDir2.map(_.response.body), Some(Some(Web.bblBlogRootContainer))) } From aea15e2c929be3b263611191e160c343e3633eb8 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Wed, 24 May 2023 15:06:32 +0200 Subject: [PATCH 32/42] MiniCV script works with cache (but not perfectly) --- build.sbt | 5 +- .../mules/http4s/CachedResponse.scala | 6 + .../cache/InterpretedCacheMiddleware.scala | 7 +- .../cache/InterpretedCacheMiddleTest.scala | 33 +-- .../http/cache/{Web.scala => WebTest.scala} | 2 +- .../run/cosy/ld/http4s/RDFDecoders.scala | 11 +- project/Dependencies.scala | 3 +- .../main/scala/scripts/AnHttpSigClient.scala | 83 +++++-- .../src/main/scala/scripts/MiniCF.scala | 2 +- .../net/bblfish/wallet/BasicAuthWallet.scala | 217 ++++++++++++------ 10 files changed, 251 insertions(+), 118 deletions(-) rename cache/shared/src/test/scala/run/cosy/http/cache/{Web.scala => WebTest.scala} (99%) diff --git a/build.sbt b/build.sbt index 53ffd70..de8188e 100644 --- a/build.sbt +++ b/build.sbt @@ -156,7 +156,7 @@ lazy val ioExt4s = crossProject(JVMPlatform).crossType(CrossType.Full).in(file(" resolvers += sonatypeSNAPSHOT, libraryDependencies ++= Seq( http4s.client.value, - banana.bananaIO.value + banana.bananaJenaIO.value ), libraryDependencies ++= Seq( munit.value % Test, @@ -200,7 +200,7 @@ lazy val wallet = crossProject(JVMPlatform) // , JSPlatform) // do I also need to run `npm install n3` ? // Compile / npmDependencies += NPM.n3, // Test / npmDependencies += NPM.n3 - ) + ).dependsOn(cache) lazy val scripts = crossProject(JVMPlatform).in(file("scripts")) // .settings( @@ -215,6 +215,7 @@ lazy val scripts = crossProject(JVMPlatform).in(file("scripts")) crypto.bobcats.value classifier ("tests"), // bobcats test examples, crypto.bobcats.value classifier ("tests-sources"), // bobcats test examples soources, other.scalaUri.value, + http4s.ember_client.value, crypto.nimbusJWT_JDK.value, crypto.bouncyJCA_JDK.value diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala index 181daa2..6739467 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala @@ -38,6 +38,12 @@ final case class CachedResponse[T]( headers, this.body ) + def map[S](f: T => S): CachedResponse[S] = new CachedResponse[S]( + this.status, + this.httpVersion, + this.headers, + this.body.map(f) + ) object CachedResponse: diff --git a/cache/shared/src/main/scala/run/cosy/http/cache/InterpretedCacheMiddleware.scala b/cache/shared/src/main/scala/run/cosy/http/cache/InterpretedCacheMiddleware.scala index 78ce940..9cc54b2 100644 --- a/cache/shared/src/main/scala/run/cosy/http/cache/InterpretedCacheMiddleware.scala +++ b/cache/shared/src/main/scala/run/cosy/http/cache/InterpretedCacheMiddleware.scala @@ -33,18 +33,21 @@ object InterpretedCacheMiddleware: def client[F[_]: Concurrent: Clock, T]( cache: TreeDirCache[F, CacheItem[T]], interpret: Response[F] => F[CachedResponse[T]], + enhance: Request[F] => Request[F] = identity, cacheType: CacheType = CacheType.Private ): Client[F] => InterpClient[F, Resource[F, *], T] = (client: Client[F]) => Kleisli( - Caching[F, T](cache, interpret, cacheType).request(Kleisli(client.run), Resource.liftK) + Caching[F, T](cache, interpret, cacheType) + .request(Kleisli(req => client.run(enhance(req))), Resource.liftK) ) def app[F[_]: Concurrent: Clock, T]( cache: TreeDirCache[F, CacheItem[T]], interpret: Response[F] => F[CachedResponse[T]], + enhance: Request[F] => Request[F] = identity, cacheType: CacheType = CacheType.Private ): HttpApp[F] => InterpClient[F, F, T] = (app: HttpApp[F]) => Kleisli( Caching[F, T](cache, interpret, cacheType) - .request(Kleisli(app.run), cats.arrow.FunctionK.id[F]) + .request(Kleisli(req => app.run(enhance(req))), cats.arrow.FunctionK.id[F]) ) diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala b/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala index 49c2fbc..7137d93 100644 --- a/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala +++ b/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala @@ -32,13 +32,13 @@ end InterpretedCacheMiddleTest class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: import InterpretedCacheMiddleTest.* test("test web") { - Web.httpRoutes[IO].orNotFound.run(Request[IO](uri = Uri(path = Root))).map { response => + WebTest.httpRoutes[IO].orNotFound.run(Request[IO](uri = Uri(path = Root))).map { response => assertEquals(response.status, Status.Ok) assertEquals( removeDate(response.headers), - Web.headers("/") + WebTest.headers("/") ) - } >> Web.httpRoutes[IO].orNotFound.run( + } >> WebTest.httpRoutes[IO].orNotFound.run( Request[IO](uri = Uri(path = Uri.Path.unsafeFromString("/people/henry/blog/2023/04/01/world-at-peace")) ) @@ -46,7 +46,7 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: assertEquals(response.status, Status.Ok) assertEquals( removeDate(response.headers), - Web.headers("world-at-peace", Some("/people/henry/blog/"), MediaType.text.plain) + WebTest.headers("world-at-peace", Some("/people/henry/blog/"), MediaType.text.plain) ) assertEquals( bytesToString(response.body.compile.toVector.unsafeRunSync()), @@ -72,9 +72,10 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: response.headers, Some(vec.mkString) ) - } + }, + enhance = _.putHeaders(Accept(MediaType.text.plain)) ) - cc = stringCacheMiddleWare(Web.httpRoutes[IO].orNotFound) + cc = stringCacheMiddleWare(WebTest.httpRoutes[IO].orNotFound) respWP1 <- cc.run(Request[IO](GET, worldPeace)) respRoot <- cc.run(Request[IO](GET, bbl)) rescounter1 <- cc.run(Request[IO](GET, counterUri)) @@ -89,7 +90,7 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: for links <- ci.response.headers.get[Link].toList link <- links.values.toList - if link.rel.contains(Web.defaultAccessContainer) + if link.rel.contains(WebTest.defaultAccessContainer) yield worldPeace.resolve(link.uri) x.headOption // there should be only one! } @@ -102,7 +103,7 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: assertEquals(respWP1.status, Status.Ok) assertEquals( removeDate(respWP1.headers), - Web.headers("world-at-peace", Some("/people/henry/blog/"), MediaType.text.plain) + WebTest.headers("world-at-peace", Some("/people/henry/blog/"), MediaType.text.plain) ) assertEquals( respWP1.body, @@ -111,11 +112,11 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: assertEquals(respRoot.status, Status.Ok) assertEquals( removeDate(respRoot.headers), - Web.headers("/") + WebTest.headers("/") ) assertEquals( respRoot.body, - Some(Web.rootTtl) + Some(WebTest.rootTtl) ) assertEquals(rescounter1.status, Status.Ok) // this tells us that we hit the cache once @@ -129,7 +130,7 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: assertEquals(respWP2.status, Status.Ok) assertEquals( removeDate(respWP1.headers), - Web.headers("world-at-peace", Some("/people/henry/blog/"), MediaType.text.plain) + WebTest.headers("world-at-peace", Some("/people/henry/blog/"), MediaType.text.plain) ) assertEquals(rescounter2.status, Status.Ok) @@ -144,7 +145,7 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: assertEquals(respRoot2.status, Status.Ok) assertEquals( respRoot2.body, - Some(Web.rootTtl) + Some(WebTest.rootTtl) ) rescounter3.body.map { body => val c1 = parseMap(body) @@ -155,17 +156,17 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: // test direct access to cache assertEquals(cachedWorldPeace.get.response.status, Status.Ok) - assertEquals(cachedWorldPeace.get.response.body, Some(Web.bblWorldAtPeace)) + assertEquals(cachedWorldPeace.get.response.body, Some(WebTest.bblWorldAtPeace)) assertEquals(cacheBlogDir.status, Status.Ok) assertEquals( removeDate(cacheBlogDir.headers), - Web.headers("") + WebTest.headers("") ) // the closest dir is the root because we have not cached the blog dir - assertEquals(closestDir1.map(_.response.body), Some(Some(Web.rootTtl))) + assertEquals(closestDir1.map(_.response.body), Some(Some(WebTest.rootTtl))) // now the blog dir is cached - assertEquals(closestDir2.map(_.response.body), Some(Some(Web.bblBlogRootContainer))) + assertEquals(closestDir2.map(_.response.body), Some(Some(WebTest.bblBlogRootContainer))) } end InterpretedCacheMiddleTest diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala b/cache/shared/src/test/scala/run/cosy/http/cache/WebTest.scala similarity index 99% rename from cache/shared/src/test/scala/run/cosy/http/cache/Web.scala rename to cache/shared/src/test/scala/run/cosy/http/cache/WebTest.scala index c6e8906..2b0b767 100644 --- a/cache/shared/src/test/scala/run/cosy/http/cache/Web.scala +++ b/cache/shared/src/test/scala/run/cosy/http/cache/WebTest.scala @@ -34,7 +34,7 @@ import cats.Monad import cats.effect.kernel.Clock // import io.chrisdavenport.mules.http4s.CachedResponse.body -object Web: +object WebTest: extension (uri: Uri) /** Uri("/") + ".acl" == Uri("/.acl") and Uri("foo")+".acl" == Uri("foo.acl") */ diff --git a/ioExt4s/shared/src/main/scala/run/cosy/ld/http4s/RDFDecoders.scala b/ioExt4s/shared/src/main/scala/run/cosy/ld/http4s/RDFDecoders.scala index 208a1de..9792e7c 100644 --- a/ioExt4s/shared/src/main/scala/run/cosy/ld/http4s/RDFDecoders.scala +++ b/ioExt4s/shared/src/main/scala/run/cosy/ld/http4s/RDFDecoders.scala @@ -39,6 +39,8 @@ import scala.util.{Failure, Left, Right, Success, Try} import org.http4s.client.Client import org.http4s.headers.Accept import org.w3.banana.RDF.rGraph +import org.http4s.Uri +import org.w3.banana.io.RDFReader class RDFDecoders[F[_], Rdf <: RDF](using val ops: Ops[Rdf], @@ -48,7 +50,7 @@ class RDFDecoders[F[_], Rdf <: RDF](using jsonLDReader: RelRDFReader[Rdf, Try, JsonLd] ): - private def decoderForRdfReader[T](mt: MediaRange, mts: MediaRange*)( + private def decoderForRelRdfReader[T](mt: MediaRange, mts: MediaRange*)( reader: RelRDFReader[Rdf, Try, T], errmsg: String ): EntityDecoder[F, rGraph[Rdf]] = @@ -63,6 +65,7 @@ class RDFDecoders[F[_], Rdf <: RDF](using } ) } + import MediaType.{application, text} import MediaType.application.`ld+json` @@ -76,18 +79,18 @@ class RDFDecoders[F[_], Rdf <: RDF](using List("nt") ) - val turtleDecoder: EntityDecoder[F, rGraph[Rdf]] = decoderForRdfReader(text.turtle, ntriples)( + val turtleDecoder: EntityDecoder[F, rGraph[Rdf]] = decoderForRelRdfReader(text.turtle, ntriples)( turtleReader, "Rdf Turtle Reader failed" ) - val rdfxmlDecoder: EntityDecoder[F, rGraph[Rdf]] = decoderForRdfReader(application.`rdf+xml`)( + val rdfxmlDecoder: EntityDecoder[F, rGraph[Rdf]] = decoderForRelRdfReader(application.`rdf+xml`)( rdfXmlReader, "Rdf Rdf/XML Reader failed" ) // val ntriplesDecoder = decoderForRdfReader(application.`n-triples`)( // ntriplesReader, "NTriples Reader failed" // ) - val jsonldDecoder: EntityDecoder[F, rGraph[Rdf]] = decoderForRdfReader(application.`ld+json`)( + val jsonldDecoder: EntityDecoder[F, rGraph[Rdf]] = decoderForRelRdfReader(application.`ld+json`)( jsonLDReader, "Json-LD Reader failed" ) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 28c14e5..cdd10bd 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -58,8 +58,9 @@ object Dependencies { // not published yet object banana { + lazy val bananaIOSync = Def.setting("net.bblfish.rdf" %%% "rdfIO-sync" % Ver.banana) lazy val bananaRdf = Def.setting("net.bblfish.rdf" %%% "banana-rdf" % Ver.banana) - lazy val bananaIO = Def.setting("net.bblfish.rdf" %%% "banana-jena-io-sync" % Ver.banana) + lazy val bananaJenaIO = Def.setting("net.bblfish.rdf" %%% "banana-jena-io-sync" % Ver.banana) lazy val bananaJena = Def.setting("net.bblfish.rdf" %%% "banana-jena-io-sync" % Ver.banana) } val scalajsDom = Def.setting("org.scala-js" %%% "scalajs-dom" % "2.0.0") diff --git a/scripts/shared/src/main/scala/scripts/AnHttpSigClient.scala b/scripts/shared/src/main/scala/scripts/AnHttpSigClient.scala index 4559af2..e917ef0 100644 --- a/scripts/shared/src/main/scala/scripts/AnHttpSigClient.scala +++ b/scripts/shared/src/main/scala/scripts/AnHttpSigClient.scala @@ -34,6 +34,15 @@ import scodec.bits.ByteVector import org.http4s.client.* import org.http4s.ember.client.* import run.cosy.http.headers.SigIn.KeyId +import org.w3.banana.RDF +import org.w3.banana.Ops +import run.cosy.http.cache.TreeDirCache.WebCache +import io.chrisdavenport.mules.http4s.CacheItem +import run.cosy.http.cache.TreeDirCache +import run.cosy.http.cache.InterpretedCacheMiddleware +import org.http4s.Response +import io.chrisdavenport.mules.http4s.CachedResponse +import fs2.io.net.Network object AnHttpSigClient: implicit val runtime: IORuntime = cats.effect.unsafe.IORuntime.global @@ -76,35 +85,63 @@ object AnHttpSigClient: lazy val signerF: IO[ByteVector => IO[ByteVector]] = Signer[IO] .build(pkcs8K, bobcats.AsymmetricKeyAlg.`rsa-pss-sha512`) - import org.w3.banana.jena.io.JenaRDFReader.given + lazy val keyIdData: KeyData[cats.effect.IO] = + new KeyData[IO](KeyId(Rfc8941.SfString(keyIdStr)), signerF) import org.w3.banana.jena.io.JenaRDFWriter.given + import org.w3.banana.jena.io.JenaRDFReader.given - lazy val keyIdData = new KeyData[IO](KeyId(Rfc8941.SfString(keyIdStr)), signerF) - - given dec: RDFDecoders[IO, R] = new RDFDecoders() import org.http4s.syntax.all.uri - given wt: WalletTools[R] = new WalletTools[R] - def ioStr(uri: H4Uri): IO[String] = emberAuthClient.flatMap(_.expect[String](uri)) + given rdfDecoders: RDFDecoders[IO, R] = RDFDecoders[IO, R] + def ioStr(uri: H4Uri, client: Client[IO]): IO[String] = ClientTools + .authClient[IO, R](keyIdData, client).flatMap(_.expect[String](uri)) - /** Ember Client able to authenticate with above keyId */ - def emberAuthClient: IO[Client[IO]] = EmberClientBuilder.default[IO].build - .use { (client: Client[IO]) => - import org.http4s.client.middleware.Logger - val loggedClient: Client[IO] = - Logger[IO](true, true, logAction = Some(str => IO(System.out.println(str))))(client) + // run "use" on this to get a client + def emberClient: Resource[IO, Client[IO]] = EmberClientBuilder.default[IO].build - val bw = new BasicWallet[IO, R]( - Map(), - Seq(keyIdData) - )(loggedClient) + def emberAuthClient: IO[Client[IO]] = emberClient.use { client => + ClientTools.authClient(keyIdData, client) + } - IO(AuthNClient[IO].apply(bw)(loggedClient)) - } - - def fetch(uriStr: String = "http://localhost:8080/protected/README"): String = - // ioStr(uri"http://localhost:8080/").unsafeRunSync() - // ioStr(uri"http://localhost:8080/protected/").unsafeRunSync() - ioStr(H4Uri.unsafeFromString(uriStr)).unsafeRunSync() + @main + def fetch(uriStr: String = "http://localhost:8080/protected/README"): Unit = + // ioStr(uri"http://localhost:8080/").unsafeRunSync() + // ioStr(uri"http://localhost:8080/protected/").unsafeRunSync() + val result = emberAuthClient.flatMap { client => + ioStr(H4Uri.unsafeFromString(uriStr), client) + }.unsafeRunSync() + println(result) end AnHttpSigClient + +object ClientTools: + import cats.effect.kernel.Clock + import cats.effect.Concurrent + import cats.syntax.all.* + + /** enhance client so that it logs and can authenticate with given keyId, and save acls to graph + * cache it creates. (a bit adhoc of a function, but it is a script) + */ + def authClient[F[_]: Clock: Async, R <: RDF]( + keyIdData: KeyData[F], + client: Client[F] + )(using + ops: Ops[R], + rdfDecoders: RDFDecoders[F, R] + ): F[Client[F]] = + import org.http4s.client.middleware.Logger + val loggedClient: Client[F] = Logger[F]( + true, + true, + logAction = Some(str => Concurrent[F].pure(System.out.println(str))) + )(client) + val walletTools: WalletTools[R] = new WalletTools[R] + for ref <- Ref.of[F, WebCache[CacheItem[RDF.rGraph[R]]]](Map.empty) + yield + val cache = TreeDirCache[F, CacheItem[RDF.rGraph[R]]](ref) + val interClientMiddleware = walletTools.cachedRelGraphMiddleware(cache) + val bw = new BasicWallet[F, R]( + Map(), + Seq(keyIdData) + )(interClientMiddleware(loggedClient)) + AuthNClient[F](bw)(loggedClient) diff --git a/scripts/shared/src/main/scala/scripts/MiniCF.scala b/scripts/shared/src/main/scala/scripts/MiniCF.scala index bde7f6d..f3a0e10 100644 --- a/scripts/shared/src/main/scala/scripts/MiniCF.scala +++ b/scripts/shared/src/main/scala/scripts/MiniCF.scala @@ -43,7 +43,7 @@ object MiniCF: given rdfDecoders: RDFDecoders[IO, JR] = new RDFDecoders[IO, JR] @main - def crawlContainer(stream: String = "http://localhost:8080/ldes/miniCityFlows/stream#"): Unit = + def crawlContainer(stream: String = "http://localhost:8080/ldes/openCF/stream#"): Unit = val streamUri: RDF.URI[JR] = ops.URI(stream) val ioStr: IO[fs2.Stream[IO, Chunk[UriNGraph[JR]]]] = AnHttpSigClient.emberAuthClient diff --git a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala index e759644..39a243b 100644 --- a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala +++ b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala @@ -49,6 +49,20 @@ import scodec.bits.ByteVector import scala.concurrent.duration.FiniteDuration import scala.reflect.TypeTest import scala.util.{Failure, Try} +import io.chrisdavenport.mules.http4s.CacheItem +import run.cosy.http.cache.TreeDirCache +import run.cosy.http.cache.InterpretedCacheMiddleware.InterpClient +import org.http4s.HttpApp +import cats.effect.kernel.Resource +import run.cosy.http.cache.InterpretedCacheMiddleware +import io.chrisdavenport.mules.http4s.CachedResponse +import cats.MonadThrow +import org.http4s.EntityDecoder +import net.bblfish.wallet.BasicWallet.defaultAC +import org.w3.banana.prefix.LDP +import run.cosy.web.util.UrlUtil.* +import io.lemonlabs.uri.AbsoluteUrl +import io.lemonlabs.uri.config.UriConfig class BasicId(val username: String, val password: String) @@ -83,9 +97,9 @@ trait ChallengeResponse: ): F[h4s.Request[F]] object BasicWallet: - val effectiveAclLink = "effectiveAccessControl" - val EffectiveAclOpt = Some(effectiveAclLink) - val aclRelTypes = List(EffectiveAclOpt, "acl", effectiveAclLink) + val defaultAC = "defaultAccessContainer" + val defaultACOpt = Some(defaultAC) + val aclRelTypes = List("acl", defaultAC) /** place code that only needs RDF and ops here. */ class WalletTools[Rdf <: RDF](using ops: Ops[Rdf]): @@ -93,9 +107,40 @@ class WalletTools[Rdf <: RDF](using ops: Ops[Rdf]): import ops.{*, given} val wac: WebACL[Rdf] = WebACL[Rdf] + val ldpContains = LDP[Rdf]("contains").toString val foaf = prefix.FOAF[Rdf] val sec: SecurityPrefix[Rdf] = SecurityPrefix[Rdf] + /** Given the signature of InterpretedCacheMiddleWare we are forced to store an rGraph in the + * cache. This is not ideal, as it means that every request on the cache will need to transform + * the rGraph into a Graph. To avoid this the client function would need to take a (response, + * uri) pair as second argument. + * + * todo: is the final URL that a redirect goes to the URL to use to absolutize a graph, or is + * the request URL after going through the redirects? Because that is important to know how much + * info is needed to be able to interpret the relative graph. + */ + def cachedRelGraphMiddleware[F[_]: Concurrent: Clock]( + cache: TreeDirCache[F, CacheItem[RDF.rGraph[Rdf]]] + )(using + rdfDecoders: RDFDecoders[F, Rdf] + ): Client[F] => InterpClient[F, Resource[F, *], RDF.rGraph[Rdf]] = InterpretedCacheMiddleware + .client[F, RDF.rGraph[Rdf]]( + cache, + (response: h4s.Response[F]) => + import rdfDecoders.allrdf + response.as[RDF.rGraph[Rdf]].map { rG => + CachedResponse[RDF.rGraph[Rdf]]( + response.status, + response.httpVersion, + response.headers, + Some(rG) + ) + } + , + enhance = _.withHeaders(rdfDecoders.allRdfAccept) + ) + def withinTry(requestUri: RDF.URI[Rdf], container: RDF.URI[Rdf]): Try[Boolean] = for case requ: ll.AbsoluteUrl <- requestUri.toLL @@ -159,15 +204,15 @@ end WalletTools * username/passwords per domain * @param keyIdDB * Key Database - * @param client - * The client needed to fetch acl resources on the web (should be a proxy) + * @param iClient + * this is an interpreted client that can fetch acls on the web and returns graphs. This allows + * us to pass in a cached interpreted client for example. */ class BasicWallet[F[_], Rdf <: RDF]( db: Map[ll.Authority, BasicId], keyIdDB: Seq[KeyData[F]] = Seq() -)(client: Client[F])(using +)(iClient: InterpClient[F, Resource[F, *], RDF.rGraph[Rdf]])(using ops: Ops[Rdf], - rdfDecoders: RDFDecoders[F, Rdf], fc: Concurrent[F], clock: Clock[F] ) extends Wallet[F]: @@ -175,7 +220,6 @@ class BasicWallet[F[_], Rdf <: RDF]( val reqSel: ReqSelectors[H4] = new ReqSelectors[H4](using new SelectorFnsH4()) import ops.{*, given} - import rdfDecoders.allrdf import reqSel.* import reqSel.RequestHd.* import run.cosy.http4s.Http4sTp @@ -206,70 +250,107 @@ class BasicWallet[F[_], Rdf <: RDF]( either.toTry end basicChallenge + // todo: do we need the return to be a Resource? Would F[Graph] be enough? + def findAclForContainer(containerUrl: AbsoluteUrl): Resource[F, RDF.Graph[Rdf]] = + // todo: should be a HEAD request!!! + iClient.run(h4s.Request(uri = containerUrl.toh4.withoutFragment)).flatMap { crG => + if crG.status.isSuccess + then findAclFor(containerUrl, crG.headers) + else Resource.raiseError(Exception(s"Unsuccessful request on $containerUrl"))(fc) + } + + def findAclFor( + requestUrl: ll.AbsoluteUrl, + responseHeaders: org.http4s.Headers + ): Resource[F, RDF.Graph[Rdf]] = responseHeaders.get[Link] match + case None => Resource + .raiseError(Exception("no Links in header. Can't find where the rules are. "))(fc) + case Some(link) => + val rels: Seq[(Either[String, String], h4s.Uri)] = link.values.toList.toSeq.collect { + case LinkValue(uri, rel, rev, _, _) if rel.isDefined || rev.isDefined => + Seq( + rel.toSeq.flatMap(_.split(" ").toSeq.map(_.asRight[String] -> uri)), + rev.toSeq.flatMap(_.split(" ").toSeq.map(_.asLeft[String] -> uri)) + ).flatten + }.flatten + // sort rels by this sequence on the first element of the tuple: defaultAC, acl, ldpcontains + // todo: if we knoew that the resource was a solid:Resource or a solid:Cointainer we could also + // proceed looking at the Url structure to see if we find the acl + // todo: if we follow links, we need to make sure we don't follow the same link twice or go in circles + val attempts: Seq[Resource[F, Either[Throwable, RDF.Graph[Rdf]]]] = rels.sortBy { + case (Left(`ldpContains`), _) => 3 + case (Right("acl"), _) => 2 + case (Right(`defaultAC`), _) => 1 + case _ => 4 + }.map((e, u) => (e, u.toLL.resolve(requestUrl, true).toAbsoluteUrl)).collect { + case (Right(`defaultAC`), uri) => findAclForContainer(uri) + case (Right("acl"), uri) => iClient.run(h4s.Request(uri = uri.toh4.withoutFragment)) + .flatMap { crG => + Resource.liftK( + if crG.status.isSuccess && crG.body.isDefined + then fc.pure(crG.body.get.resolveAgainst(uri)) + else fc.raiseError(Exception(s"Could not find ACL at $uri")) + ) + } + case (Left(`ldpContains`), uri) => findAclForContainer(uri) + }.map(_.attempt) + val w: F[Option[RDF.Graph[Rdf]]] = attempts.map(r => r.use(fc.pure)) + .collectFirstSomeM(_.map(_.toOption)) + Resource.eval(w).flatMap { + case None => Resource.raiseError(Exception("no usable Link in header."))(fc) + case Some(g) => Resource.pure(g) + } + end findAclFor + + /** given the original request and a response, return the correctly signed original request (test + * for the HttpSig WWW-Authenticate header has been done before calling this function) + */ def httpSigChallenge( - requestUrl: ll.AbsoluteUrl, // http4s.Request objects are not guaranteed to contain absolute urls + lastReqUrl: ll.AbsoluteUrl, // http4s.Request objects are not guaranteed to contain absolute urls originalRequest: h4s.Request[F], response: h4s.Response[F], nel: NonEmptyList[h4s.Challenge] ): F[h4s.Request[F]] = import BasicWallet.* - val aclLinks: List[LinkValue] = - for - httpSig <- nel.find(_.scheme == "HttpSig").toList - link <- response.headers.get[Link].toList - linkVal <- link.values.toList - rel <- linkVal.rel.toList - if aclRelTypes.contains(rel) - yield linkVal - aclLinks.find(_.rel == EffectiveAclOpt).orElse(aclLinks.headOption) match - case None => fc - .raiseError(Exception("no acl Link in header. Cannot find where the rules are.")) - case Some(linkVal) => - val h4req = llUrltoHttp4s(requestUrl) - val absLink: h4s.Uri = h4req.resolve(linkVal.uri) - client.fetchAs[RDF.rGraph[Rdf]]( - h4s.Request( - uri = absLink.withoutFragment - ) - ).flatMap { (rG: RDF.rGraph[Rdf]) => - // todo: what if original url is relative? - val lllink = http4sUrlToLLUrl(absLink).toAbsoluteUrl - val g: RDF.Graph[Rdf] = rG.resolveAgainst(lllink) - val reqRes = ops.URI(originalRequest.uri.toString) // <-- - // this requires all the info to be in the same graph. Needs generalisation to - // jump across graphs - val keyNodes: Iterator[St.Subject[Rdf]] = - for - agentNode <- findAgents(g, reqRes, originalRequest.method) - controllerTriple <- g.find(*, sec.controller, agentNode) - yield controllerTriple.subj - - import run.cosy.http4s.Http4sTp.given - val keys: Iterable[KeyData[F]] = keyNodes.collect { case u: RDF.URI[Rdf] => - keyIdDB.find(kid => kid.keyIdAtt.value.asciiStr == u.value).toList - }.flatten.to(Iterable) + val result: Resource[F, h4s.Request[F]] = findAclFor(lastReqUrl, response.headers) + .flatMap { (aclGr: RDF.Graph[Rdf]) => + import io.lemonlabs.uri.config.UriConfig + val reqRes = ops.URI(lastReqUrl.copy(fragment = None)(UriConfig.default)) + val keyNodes: Iterator[St.Subject[Rdf]] = + for + agentNode <- findAgents(aclGr, reqRes, originalRequest.method) + controllerTriple <- aclGr.find(*, sec.controller, agentNode) + yield controllerTriple.subj - for - keydt <- fc.fromOption[KeyData[F]]( - keys.headOption, - Exception( - s"none of our keys fit the ACL $lllink for resource $reqRes accessed in " + - s"${originalRequest.method} matches the rules in { $g } " - ) - ) - signingFn <- keydt.signer - now <- clock.realTime // <- todo, add clock time caching perhaps - signedReq <- MessageSignature.withSigInput[F, H4]( - originalRequest.asInstanceOf[Http.Request[H4]], - Rfc8941.Token("sig1"), - keydt.mkSigInput(now), - signingFn - ) - yield - val res = run.cosy.http4s.Http4sTp.hOps - .addHeader[Http.Request[H4]](signedReq)("Authorization", "HttpSig proof=sig1") - h4ReqToHttpReq(res) - } + import run.cosy.http4s.Http4sTp.given + val keys: Iterable[KeyData[F]] = keyNodes.collect { case u: RDF.URI[Rdf] => + keyIdDB.find(kid => kid.keyIdAtt.value.asciiStr == u.value).toList + }.flatten.to(Iterable) + + val x: F[h4s.Request[F]] = + for + keydt <- fc.fromOption[KeyData[F]]( + keys.headOption, + Exception( + s"none of our keys fit the ACL for resource $lastReqUrl accessed in " + + s"${originalRequest.method} matches the rules in graph { $aclGr } " + ) + ) + signingFn <- keydt.signer + now <- clock.realTime // <- todo, add clock time caching perhaps + signedReq <- MessageSignature.withSigInput[F, H4]( + originalRequest.asInstanceOf[Http.Request[H4]], + Rfc8941.Token("sig1"), + keydt.mkSigInput(now), + signingFn + ) + yield + val res = run.cosy.http4s.Http4sTp.hOps + .addHeader[Http.Request[H4]](signedReq)("Authorization", "HttpSig proof=sig1") + h4ReqToHttpReq(res) + Resource.liftK(x) + } + result.use(fc.pure) end httpSigChallenge /** This is different from middleware such as FollowRedirects, as that essentially continues the @@ -293,10 +374,10 @@ class BasicWallet[F[_], Rdf <: RDF]( ) case Some(h4hdr.`WWW-Authenticate`(nel)) => // do we recognise a method? for - url <- fc.fromTry(Try(http4sUrlToLLUrl(lastReq.uri).toAbsoluteUrl)) - authdReq <- fc.fromTry(basicChallenge(url.authority, lastReq, nel)) + lastReqUrl <- fc.fromTry(Try(lastReq.uri.toLL.toAbsoluteUrl)) + authdReq <- fc.fromTry(basicChallenge(lastReqUrl.authority, lastReq, nel)) .handleErrorWith { _ => - httpSigChallenge(url, lastReq, failed, nel) + httpSigChallenge(lastReqUrl, lastReq, failed, nel) } yield authdReq case _ => ??? // fail From 611dbf00e391eb84ad604ae33559e9e816f08716 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Thu, 25 May 2023 07:53:25 +0200 Subject: [PATCH 33/42] test cache non 2xx, simplify acl link following --- .../cache/InterpretedCacheMiddleTest.scala | 41 ++++- .../scala/run/cosy/http/cache/WebTest.scala | 38 +++-- test/shared/src/main/scala/mules/test.scala | 142 ++++++++++++++++++ .../net/bblfish/wallet/BasicAuthWallet.scala | 129 ++++++++++------ 4 files changed, 276 insertions(+), 74 deletions(-) create mode 100644 test/shared/src/main/scala/mules/test.scala diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala b/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala index 7137d93..7375d47 100644 --- a/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala +++ b/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala @@ -15,6 +15,8 @@ import cats.data.Kleisli import io.chrisdavenport.mules.http4s.CachedResponse import io.chrisdavenport.mules.http4s.CacheItem import org.typelevel.ci.CIStringSyntax +import cats.data.NonEmptyList +import io.chrisdavenport.mules.http4s.internal.Caching object InterpretedCacheMiddleTest: def bytesToString(bytes: Vector[Byte]): String = bytes.map(_.toChar).mkString @@ -54,6 +56,7 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: ) } } + /** we need dates so that caching can work but it is tricky to compare them */ def removeDate(headers: Headers): Headers = headers .transform(l => l.filterNot(_.name == ci"Date")) @@ -73,7 +76,7 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: Some(vec.mkString) ) }, - enhance = _.putHeaders(Accept(MediaType.text.plain)) + enhance = _.putHeaders(Accept(MediaType.text.plain)) ) cc = stringCacheMiddleWare(WebTest.httpRoutes[IO].orNotFound) respWP1 <- cc.run(Request[IO](GET, worldPeace)) @@ -85,7 +88,7 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: rescounter3 <- cc.run(Request[IO](GET, counterUri)) cachedWorldPeace <- cache.lookup(worldPeace) closestDir1 <- cache.findClosest(bblBlogVirgin)(_.isDefined) - wpLink = cachedWorldPeace.flatMap { ci => + wpAcrLink = cachedWorldPeace.flatMap { ci => val x: List[Uri] = for links <- ci.response.headers.get[Link].toList @@ -94,11 +97,17 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: yield worldPeace.resolve(link.uri) x.headOption // there should be only one! } - blogAcrDirUrl <- IO.fromOption(wpLink)( + blogDirUrl <- IO.fromOption(wpAcrLink)( new Exception(s"no default container Link header in $cachedWorldPeace") ) - cacheBlogDir <- cc.run(Request[IO](Method.GET, blogAcrDirUrl)) + cacheBlogDir <- cc.run(Request[IO](Method.GET, blogDirUrl)) + blogDirAcr = cacheBlogDir.headers.get[Link].toList.flatMap(_.values.toList) + .collectFirst { case LinkValue(acl, Some("acl"), _, _, _) => acl } + .map(u => blogDirUrl.resolve(u)) + x = assert(blogDirAcr.nonEmpty, s"no acl link header in $cacheBlogDir") closestDir2 <- cache.findClosest(bblBlogVirgin)(_.isDefined) + cachedBlogDirAcl <- cc.run(Request[IO](Method.GET, blogDirAcr.get)) + cachedBlogDir2 <- cache.lookup(blogDirUrl) yield assertEquals(respWP1.status, Status.Ok) assertEquals( @@ -158,15 +167,35 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: assertEquals(cachedWorldPeace.get.response.status, Status.Ok) assertEquals(cachedWorldPeace.get.response.body, Some(WebTest.bblWorldAtPeace)) - assertEquals(cacheBlogDir.status, Status.Ok) + assertEquals(cacheBlogDir.status, Status.NotFound) assertEquals( removeDate(cacheBlogDir.headers), - WebTest.headers("") + Headers( + Link( + LinkValue(Uri(path = Uri.Path(Vector(Uri.Path.Segment(".acr")))), rel = Some("acl")) + ) + ) ) // the closest dir is the root because we have not cached the blog dir assertEquals(closestDir1.map(_.response.body), Some(Some(WebTest.rootTtl))) // now the blog dir is cached assertEquals(closestDir2.map(_.response.body), Some(Some(WebTest.bblBlogRootContainer))) + // we got the acl for the blog dir after following the link header of a 401 resource + assertEquals(cachedBlogDirAcl.status, Status.Ok) + assertEquals(cachedBlogDirAcl.body, Some(WebTest.bblBlogAcr)) + + // the cache should have the same content as the response, even when returning a 404 + // (ie, we did not have to make a new request to the web) + assertEquals(cachedBlogDir2.get.response.status, Status.NotFound) + assertEquals( + removeDate(cachedBlogDir2.get.response.headers), + Headers( + Link( + LinkValue(Uri(path = Uri.Path(Vector(Uri.Path.Segment(".acr")))), rel = Some("acl")) + ) + ) + ) + } end InterpretedCacheMiddleTest diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/WebTest.scala b/cache/shared/src/test/scala/run/cosy/http/cache/WebTest.scala index 2b0b767..ea9e4a4 100644 --- a/cache/shared/src/test/scala/run/cosy/http/cache/WebTest.scala +++ b/cache/shared/src/test/scala/run/cosy/http/cache/WebTest.scala @@ -60,7 +60,7 @@ object WebTest: ), Allow(GET, HEAD) ) - + val defaultAccessContainer = "defaultAccessContainer" val rootDir = Uri(path = Root) // relative URLs @@ -116,11 +116,7 @@ object WebTest: | wac:default <.> . |""".stripMargin - val bblBlogRootContainer = """ - |@prefix ldp: . - |<> a ldp:BasicContainer; - | . ldp:contains <2023/> . - |""".stripMargin + val bblBlogRootContainer = "".stripMargin val bblWorldAtPeace = "Hello World!" val bblBlogVirgin = "Play in three acts" @@ -130,12 +126,12 @@ object WebTest: ): HttpRoutes[F] = val counter = AtomicReference(Map.empty[Uri.Path, Int]) val inc = Kleisli[OptionT[F, *], Request[F], Request[F]] { req => - OptionT.liftF(AS.delay { - counter.updateAndGet { m => - val count = m.getOrElse(req.uri.path, 0) - m.updated(req.uri.path, count + 1) - } - }) >> OptionT.pure(req) + OptionT.liftF(AS.delay { + counter.updateAndGet { m => + val count = m.getOrElse(req.uri.path, 0) + m.updated(req.uri.path, count + 1) + } + }) >> OptionT.pure(req) } val routes: Kleisli[OptionT[F, *], Request[F], Response[F]] = HttpRoutes.of[F] { case GET -> Root => AS.pure( @@ -159,19 +155,21 @@ object WebTest: headers = headers("card", Some("/")) ) ) - case GET -> Root / "people" / "henry" / "blog" / "2023" / "05" / "18" / "birth" => - OK[F]( + case GET -> Root / "people" / "henry" / "blog" / "2023" / "05" / "18" / "birth" => OK[F]( bblBlogVirgin, headers("birth", Some("/people/henry/blog/"), MediaType.text.plain) ) case GET -> Root / "people" / "henry" / "blog" / "" => // <- is this "ends with slash"? + Async[F].pure( + Response[F]( + status = Status.NotFound, + entity = Entity.empty, + headers = Headers(Link(LinkValue(Uri(path = Uri.Path(Vector(Uri.Path.Segment(".acr")))), rel = Some("acl")))) + ) + ) + case GET -> Root / "people" / "henry" / "blog" / "2023" / "04" / "01" / "world-at-peace" => OK[F]( - bblBlogRootContainer, - headers("") - ) - case GET -> Root / "people" / "henry" / "blog" / "2023" / "04" / "01" / "world-at-peace" => - OK[F]( - bblWorldAtPeace, + bblWorldAtPeace, headers("world-at-peace", Some("/people/henry/blog/"), MediaType.text.plain) ) case GET -> Root / "people" / "henry" / "blog" / ".acr" => OK[F]( diff --git a/test/shared/src/main/scala/mules/test.scala b/test/shared/src/main/scala/mules/test.scala new file mode 100644 index 0000000..b9dc801 --- /dev/null +++ b/test/shared/src/main/scala/mules/test.scala @@ -0,0 +1,142 @@ +/* + * Copyright 2021 bblfish.net + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mules + +object test: + import cats.* + import cats.implicits.* + import cats.effect.* + import io.chrisdavenport.mules.* + import io.chrisdavenport.mules.caffeine.* + import io.chrisdavenport.mules.http4s.* + import org.http4s.* + import org.http4s.implicits.* + import org.http4s.client.Client + import org.http4s.ember.client.EmberClientBuilder + + def testMiddleware[F[_]: Concurrent](c: Client[F], ref: Ref[F, Int]): Client[F] = Client { req => + c.run(req).evalMap(resp => ref.update(_ + 1).as(resp)) + } + + val jQueryRequest = + Request[IO](Method.GET, uri"https://code.jquery.com/jquery-3.4.1.slim.min.js") + // jQueryRequest: Request[[A >: Nothing <: Any] => IO[A]] = ( + // = GET, + // = Uri( + // scheme = Some(value = Scheme(https)), + // authority = Some( + // value = Authority( + // userInfo = None, + // host = RegName(host = code.jquery.com), + // port = None + // ) + // ), + // path = /jquery-3.4.1.slim.min.js, + // query = , + // fragment = None + // ), + // = HttpVersion(major = 1, minor = 1), + // = Headers(), + // = Stream(..), + // = org.typelevel.vault.Vault@7662cebd + // ) + + val exampleCached: IO[(Int, Int)] = EmberClientBuilder.default[IO].build.use { client => + for + cache <- CaffeineCache.build[IO, (Method, Uri), CacheItem](None, None, 10000L.some) + counter <- Ref[IO].of(0) + cacheMiddleware = CacheMiddleware.client(cache, CacheType.Public) + finalClient = cacheMiddleware(testMiddleware(client, counter)) + _ <- finalClient.run(jQueryRequest).use(_.as[String]) + count1 <- counter.get + _ <- finalClient.run(jQueryRequest).use(_.as[String]) + count2 <- counter.get + yield (count1, count2) + } + // exampleCached: IO[Tuple2[Int, Int]] = FlatMap( + // ioe = Attempt( + // ioa = Map( + // ioe = Blocking( + // hint = Blocking, + // thunk = fs2.io.net.tls.TLSContextCompanionPlatform$BuilderCompanionPlatform$AsyncBuilder$$Lambda$12570/0x00000008033a0440@72b94e70, + // event = cats.effect.tracing.TracingEvent$StackTrace + // ), + // f = fs2.io.net.tls.TLSContextCompanionPlatform$BuilderCompanionPlatform$AsyncBuilder$$Lambda$12572/0x00000008033a2040@1aed8e3c, + // event = cats.effect.tracing.TracingEvent$StackTrace + // ) + // ), + // f = cats.effect.kernel.Resource$$Lambda$12577/0x00000008033a6040@7a9abcfa, + // event = cats.effect.tracing.TracingEvent$StackTrace + // ) + + import cats.effect.unsafe.implicits.global // DON'T DO THIS IN PROD // DON'T DO THIS IN PROD + + exampleCached.unsafeRunSync() + // res0: Tuple2[Int, Int] = (1, 1) + + val dadJokesRequest = Request[IO](Method.GET, uri"https://icanhazdadjoke.com/") + // dadJokesRequest: Request[[A >: Nothing <: Any] => IO[A]] = ( + // = GET, + // = Uri( + // scheme = Some(value = Scheme(https)), + // authority = Some( + // value = Authority( + // userInfo = None, + // host = RegName(host = icanhazdadjoke.com), + // port = None + // ) + // ), + // path = /, + // query = , + // fragment = None + // ), + // = HttpVersion(major = 1, minor = 1), + // = Headers(), + // = Stream(..), + // = org.typelevel.vault.Vault@719e680c + // ) + + val exampleUnCached = EmberClientBuilder.default[IO].build.use { client => + for + cache <- CaffeineCache.build[IO, (Method, Uri), CacheItem](None, None, 10000L.some) + counter <- Ref[IO].of(0) + cacheMiddleware = CacheMiddleware.client(cache, CacheType.Public) + finalClient = cacheMiddleware(testMiddleware(client, counter)) + _ <- finalClient.run(dadJokesRequest).use(_.as[String]) + count1 <- counter.get + _ <- finalClient.run(dadJokesRequest).use(_.as[String]) + count2 <- counter.get + yield (count1, count2) + } + // exampleUnCached: IO[Tuple2[Int, Int]] = FlatMap( + // ioe = Attempt( + // ioa = Map( + // ioe = Blocking( + // hint = Blocking, + // thunk = fs2.io.net.tls.TLSContextCompanionPlatform$BuilderCompanionPlatform$AsyncBuilder$$Lambda$12570/0x00000008033a0440@72b94e70, + // event = cats.effect.tracing.TracingEvent$StackTrace + // ), + // f = fs2.io.net.tls.TLSContextCompanionPlatform$BuilderCompanionPlatform$AsyncBuilder$$Lambda$12572/0x00000008033a2040@5a74133e, + // event = cats.effect.tracing.TracingEvent$StackTrace + // ) + // ), + // f = cats.effect.kernel.Resource$$Lambda$12577/0x00000008033a6040@506cf910, + // event = cats.effect.tracing.TracingEvent$StackTrace + // ) + + exampleUnCached.unsafeRunSync() +// res1: Tuple2[Int, Int] = (1, 2) diff --git a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala index 39a243b..7dc9988 100644 --- a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala +++ b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala @@ -63,6 +63,8 @@ import org.w3.banana.prefix.LDP import run.cosy.web.util.UrlUtil.* import io.lemonlabs.uri.AbsoluteUrl import io.lemonlabs.uri.config.UriConfig +import BasicWallet.* +import org.http4s.Method class BasicId(val username: String, val password: String) @@ -99,7 +101,38 @@ trait ChallengeResponse: object BasicWallet: val defaultAC = "defaultAccessContainer" val defaultACOpt = Some(defaultAC) - val aclRelTypes = List("acl", defaultAC) + val ldpContains = "http://www.w3.org/ns/ldp#contains" + + /** extract the links from the headers as pairs of Eithers for rev or rel relations and the url + * value, and absolutize the Url. Todo: we use ll.Uri here, because they have a clear type for + * absolute urls, but this is a we really need a url abstraction that does this right + */ + def extractLinks( + requestUrl: AbsoluteUrl, + reponseHeaders: org.http4s.Headers + ): Seq[(Either[String, String], ll.AbsoluteUrl)] = reponseHeaders.get[Link] match + case None => Seq() + case Some(link) => link.values.toList.toSeq.collect { + case LinkValue(uri, rel, rev, _, _) if rel.isDefined || rev.isDefined => + Seq( + rel.toSeq.flatMap(_.split(" ").toSeq.map(_.asRight[String] -> uri)), + rev.toSeq.flatMap(_.split(" ").toSeq.map(_.asLeft[String] -> uri)) + ).flatten + }.flatten.map((e, u) => (e, u.toLL.resolve(requestUrl, true).toAbsoluteUrl)) + + /** valuees for sorting priority of link relations + * + * + todo: if we knoew that the resource was a solid:Resource or a solid:Cointainer we could + * also proceed looking at the Url structure to see if we find the acl + todo: if we follow + * links, we need to make sure we don't follow the same link twice or go in circles + */ + def relPriority(rel: Either[String, String]): Int = rel match + case Left(`ldpContains`) => 3 + case Right("acl") => 2 + case Right(`defaultAC`) => 1 + case _ => 4 + +end BasicWallet /** place code that only needs RDF and ops here. */ class WalletTools[Rdf <: RDF](using ops: Ops[Rdf]): @@ -107,7 +140,6 @@ class WalletTools[Rdf <: RDF](using ops: Ops[Rdf]): import ops.{*, given} val wac: WebACL[Rdf] = WebACL[Rdf] - val ldpContains = LDP[Rdf]("contains").toString val foaf = prefix.FOAF[Rdf] val sec: SecurityPrefix[Rdf] = SecurityPrefix[Rdf] @@ -128,15 +160,24 @@ class WalletTools[Rdf <: RDF](using ops: Ops[Rdf]): .client[F, RDF.rGraph[Rdf]]( cache, (response: h4s.Response[F]) => - import rdfDecoders.allrdf - response.as[RDF.rGraph[Rdf]].map { rG => - CachedResponse[RDF.rGraph[Rdf]]( + if !response.status.isSuccess + then + Concurrent[F].pure(CachedResponse[RDF.rGraph[Rdf]]( response.status, response.httpVersion, response.headers, - Some(rG) - ) - } + None + )) + else + import rdfDecoders.allrdf + response.as[RDF.rGraph[Rdf]].map { rG => + CachedResponse[RDF.rGraph[Rdf]]( + response.status, + response.httpVersion, + response.headers, + Some(rG) + ) + } , enhance = _.withHeaders(rdfDecoders.allRdfAccept) ) @@ -251,55 +292,47 @@ class BasicWallet[F[_], Rdf <: RDF]( end basicChallenge // todo: do we need the return to be a Resource? Would F[Graph] be enough? + /** + * Fetches the ACL for the given uri, and if it is not found, tries to find the ACL for the + * linkedToContainer (not implemented yet). + */ + def fetchAclGr( + uri: AbsoluteUrl, + fallbackContainer: Option[ll.AbsoluteUrl] = None + ): Resource[F, RDF.Graph[Rdf]] = iClient.run(h4s.Request(uri = uri.toh4.withoutFragment)) + .flatMap { crG => + Resource.liftK( + if crG.status.isSuccess && crG.body.isDefined + then fc.pure(crG.body.get.resolveAgainst(uri)) + else fc.raiseError(Exception(s"Could not find ACL at $uri")) + ) + } + def findAclForContainer(containerUrl: AbsoluteUrl): Resource[F, RDF.Graph[Rdf]] = // todo: should be a HEAD request!!! - iClient.run(h4s.Request(uri = containerUrl.toh4.withoutFragment)).flatMap { crG => - if crG.status.isSuccess - then findAclFor(containerUrl, crG.headers) - else Resource.raiseError(Exception(s"Unsuccessful request on $containerUrl"))(fc) + iClient.run(h4s.Request(Method.HEAD, uri = containerUrl.toh4.withoutFragment)).flatMap { crG => + // todo: potentially check for the right status codes... + // which ones would be correct? Clearly 201, 202, 404. But what others? + extractLinks(containerUrl, crG.headers).collectFirst { case (Right("acl"), uri) => + fetchAclGr(uri) + }.getOrElse { + Resource.raiseError[F,RDF.Graph[Rdf],Throwable](Throwable(s"Could not find ACL for $containerUrl")) + } + } def findAclFor( requestUrl: ll.AbsoluteUrl, responseHeaders: org.http4s.Headers - ): Resource[F, RDF.Graph[Rdf]] = responseHeaders.get[Link] match - case None => Resource - .raiseError(Exception("no Links in header. Can't find where the rules are. "))(fc) - case Some(link) => - val rels: Seq[(Either[String, String], h4s.Uri)] = link.values.toList.toSeq.collect { - case LinkValue(uri, rel, rev, _, _) if rel.isDefined || rev.isDefined => - Seq( - rel.toSeq.flatMap(_.split(" ").toSeq.map(_.asRight[String] -> uri)), - rev.toSeq.flatMap(_.split(" ").toSeq.map(_.asLeft[String] -> uri)) - ).flatten - }.flatten - // sort rels by this sequence on the first element of the tuple: defaultAC, acl, ldpcontains - // todo: if we knoew that the resource was a solid:Resource or a solid:Cointainer we could also - // proceed looking at the Url structure to see if we find the acl - // todo: if we follow links, we need to make sure we don't follow the same link twice or go in circles - val attempts: Seq[Resource[F, Either[Throwable, RDF.Graph[Rdf]]]] = rels.sortBy { - case (Left(`ldpContains`), _) => 3 - case (Right("acl"), _) => 2 - case (Right(`defaultAC`), _) => 1 - case _ => 4 - }.map((e, u) => (e, u.toLL.resolve(requestUrl, true).toAbsoluteUrl)).collect { + ): Resource[F, RDF.Graph[Rdf]] = + val links = extractLinks(requestUrl, responseHeaders).sortBy(x => relPriority(x._1)) + val default = links.collectFirst { case (Right(`defaultAC`), uri) => findAclForContainer(uri) - case (Right("acl"), uri) => iClient.run(h4s.Request(uri = uri.toh4.withoutFragment)) - .flatMap { crG => - Resource.liftK( - if crG.status.isSuccess && crG.body.isDefined - then fc.pure(crG.body.get.resolveAgainst(uri)) - else fc.raiseError(Exception(s"Could not find ACL at $uri")) - ) - } - case (Left(`ldpContains`), uri) => findAclForContainer(uri) - }.map(_.attempt) - val w: F[Option[RDF.Graph[Rdf]]] = attempts.map(r => r.use(fc.pure)) - .collectFirstSomeM(_.map(_.toOption)) - Resource.eval(w).flatMap { - case None => Resource.raiseError(Exception("no usable Link in header."))(fc) - case Some(g) => Resource.pure(g) + case (Right("acl"), uri) => fetchAclGr(uri) } + default.getOrElse { + Resource.raiseError[F, RDF.Graph[Rdf],Throwable](Exception(s"No useable link to find ACL from $requestUrl")) + } end findAclFor /** given the original request and a response, return the correctly signed original request (test From d0e69d5bfde5c695fa438a46253f676195ff5105 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Fri, 26 May 2023 17:54:28 +0200 Subject: [PATCH 34/42] Caching of 401 responses & support for HEAD --- .../mules/http4s/CacheItem.scala | 5 +- .../mules/http4s/CachedResponse.scala | 15 +- .../mules/http4s/internal/CacheRules.scala | 94 +++++++----- .../mules/http4s/internal/Caching.scala | 11 +- .../cache/InterpretedCacheMiddleTest.scala | 139 +++++++++++++----- .../scala/run/cosy/http/cache/WebTest.scala | 113 ++++++++------ 6 files changed, 243 insertions(+), 134 deletions(-) diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala index 15a561e..ca0e59d 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala @@ -22,11 +22,13 @@ import cats.* import cats.effect.* import cats.implicits.* import org.http4s.HttpDate +import org.http4s.Method /** Cache Items are what we place in the cache, this is exposed so that caches can be constructed by * the user for this type */ final case class CacheItem[T]( + requestMethod: Method, created: HttpDate, expires: Option[HttpDate], response: CachedResponse[T] @@ -35,9 +37,10 @@ final case class CacheItem[T]( object CacheItem: def create[F[_]: Clock: MonadThrow, T]( + requestMethod: Method, response: CachedResponse[T], expires: Option[HttpDate] - ): F[CacheItem[T]] = HttpDate.current[F].map(date => new CacheItem(date, expires, response)) + ): F[CacheItem[T]] = HttpDate.current[F].map(date => new CacheItem(requestMethod, date, expires, response)) private[http4s] final case class Age(val deltaSeconds: Long) extends AnyVal private[http4s] object Age: diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala index 6739467..67f69fd 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala @@ -32,18 +32,9 @@ final case class CachedResponse[T]( headers: Headers, body: Option[T] ): - def withHeaders(headers: Headers): CachedResponse[T] = new CachedResponse[T]( - this.status, - this.httpVersion, - headers, - this.body - ) - def map[S](f: T => S): CachedResponse[S] = new CachedResponse[S]( - this.status, - this.httpVersion, - this.headers, - this.body.map(f) - ) + def withHeaders(headers: Headers): CachedResponse[T] = this.copy(headers = headers) + + def map[S](f: T => S): CachedResponse[S] = this.copy(body = body.map(f)) object CachedResponse: diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/CacheRules.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/CacheRules.scala index d624b5d..d5cb3bf 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/CacheRules.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/CacheRules.scala @@ -29,6 +29,7 @@ import cats.implicits.* import cats.data.* import org.typelevel.ci.* +// see Caching RFC https://www.rfc-editor.org/rfc/rfc9111.html private[http4s] object CacheRules: def requestCanUseCached[F[_]](req: Request[F]): Boolean = methodIsCacheable(req.method) && @@ -55,6 +56,8 @@ private[http4s] object CacheRules: Status.MultipleChoices, // 300 Status.MovedPermanently, // 301 // Status.NotModified , // 304 + Status.Unauthorized, // 401 -- cache should be overriden by successful auth request + // Status.PaymentRequired, // 402 -- cache should be overriden by successful payment Status.NotFound, // 404 Status.MethodNotAllowed, // 405 Status.Gone, // 410 @@ -64,42 +67,61 @@ private[http4s] object CacheRules: def statusIsCacheable(s: Status): Boolean = cacheableStatus.contains(s) - def cacheAgeAcceptable[F[_]](req: Request[F], item: CacheItem[?], now: HttpDate): Boolean = - req.headers.get[`Cache-Control`] match - case None => - // TODO: Investigate how this check works with cache-control - // If the data in the cache is expired and client does not explicitly - // accept stale data, then age is not ok. - item.expires.map(expiresAt => expiresAt >= now).getOrElse(true) - case Some(`Cache-Control`(values)) => - val age = CacheItem.Age.of(item.created, now) - val lifetime = CacheItem.CacheLifetime.of(item.expires, now) - - val maxAgeMet: Boolean = values.toList - .collectFirst { case c @ CacheDirective.`max-age`(_) => c } - .map(maxAge => age.deltaSeconds.seconds <= maxAge.deltaSeconds).getOrElse(true) - - val maxStaleMet: Boolean = { - for - maxStale <- values.toList.collectFirst { case c @ CacheDirective.`max-stale`(_) => - c.deltaSeconds - }.flatten - stale <- lifetime - yield if stale.deltaSeconds >= 0 then true else stale.deltaSeconds.seconds <= maxStale - }.getOrElse(true) - - val minFreshMet: Boolean = { - for - minFresh <- values.toList.collectFirst { case CacheDirective.`min-fresh`(seconds) => - seconds - } - expiresAt <- item.expires - yield (expiresAt.epochSecond - now.epochSecond).seconds <= minFresh - }.getOrElse(true) - - // println(s"Age- $age, Lifetime- $lifetime, maxAgeMet: $maxAgeMet, maxStaleMet: $maxStaleMet, minFreshMet: $minFreshMet") - - maxAgeMet && maxStaleMet && minFreshMet + def cachedObjectOk[F[_], T]( + req: Request[F], + item: CacheItem[T], + now: HttpDate + ): Option[CacheItem[T]] = + if cacheAgeAcceptable(req, item, now) && !requestOverrides401InCache(req, item) then + if req.method == item.requestMethod then Some(item) + else if req.method == Method.HEAD && item.requestMethod == Method.GET then + Some(item.copy(response = item.response.copy(body = None))) + else None + else None + + private def requestOverrides401InCache[F[_]](req: Request[F], item: CacheItem[?]): Boolean = + req.method == Method.GET && item.response.status == Status.Unauthorized && req.headers + .get[Authorization].isDefined + + private def cacheAgeAcceptable[F[_]]( + req: Request[F], + item: CacheItem[?], + now: HttpDate + ): Boolean = req.headers.get[`Cache-Control`] match + case None => + // TODO: Investigate how this check works with cache-control + // If the data in the cache is expired and client does not explicitly + // accept stale data, then age is not ok. + item.expires.map(expiresAt => expiresAt >= now).getOrElse(true) + case Some(`Cache-Control`(values)) => + val age = CacheItem.Age.of(item.created, now) + val lifetime = CacheItem.CacheLifetime.of(item.expires, now) + + val maxAgeMet: Boolean = values.toList.collectFirst { case c @ CacheDirective.`max-age`(_) => + c + }.map(maxAge => age.deltaSeconds.seconds <= maxAge.deltaSeconds).getOrElse(true) + + val maxStaleMet: Boolean = { + for + maxStale <- values.toList.collectFirst { case c @ CacheDirective.`max-stale`(_) => + c.deltaSeconds + }.flatten + stale <- lifetime + yield if stale.deltaSeconds >= 0 then true else stale.deltaSeconds.seconds <= maxStale + }.getOrElse(true) + + val minFreshMet: Boolean = { + for + minFresh <- values.toList.collectFirst { case CacheDirective.`min-fresh`(seconds) => + seconds + } + expiresAt <- item.expires + yield (expiresAt.epochSecond - now.epochSecond).seconds <= minFresh + }.getOrElse(true) + + // println(s"Age- $age, Lifetime- $lifetime, maxAgeMet: $maxAgeMet, maxStaleMet: $maxStaleMet, minFreshMet: $minFreshMet") + + maxAgeMet && maxStaleMet && minFreshMet def onlyIfCached[F[_]](req: Request[F]): Boolean = req.headers.get[`Cache-Control`].exists { _.values.exists { diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala index b9bc707..2f8a3e9 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala @@ -49,10 +49,11 @@ case class Caching[F[_]: Concurrent: Clock, T]( if CacheRules.onlyIfCached(req) then fk(CachedResponse.errorResponse[T](Status.GatewayTimeout).pure[F]) else app.run(req).flatMap(resp => fk(withResponse(req, resp))) - case Some(item) => - if CacheRules.cacheAgeAcceptable(req, item, now) then fk(item.response.pure[F]) - else - app.run( + case Some(item) => CacheRules.cachedObjectOk(req, item, now) match + case Some(item) => + fk(item.response.pure[F]) + case None => + app.run( req.putHeaders( CacheRules.getIfMatch(item.response).map(modelledHeadersToRaw(_)).toSeq* ).putHeaders( @@ -83,7 +84,7 @@ case class Caching[F[_]: Concurrent: Clock, T]( case _ => interpret(resp) now <- HttpDate.current[F] expires = CacheRules.FreshnessAndExpiration.getExpires(now, resp) - item <- CacheItem.create(cachedResp, expires.some) + item <- CacheItem.create(req.method, cachedResp, expires.some) _ <- cache.insert(req.uri, item) yield cachedResp else interpret(resp) diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala b/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala index 7375d47..c66e161 100644 --- a/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala +++ b/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala @@ -28,7 +28,27 @@ object InterpretedCacheMiddleTest: val bblBlogVirgin: Uri = Uri .unsafeFromString("https://bblfish.net/people/henry/blog/2023/05/18/birth") val bbl: Uri = Uri.unsafeFromString("https://bblfish.net/") + val bblBlogDir: Uri = Uri.unsafeFromString("https://bblfish.net/people/henry/blog/") val counterUri = Uri.unsafeFromString("https://bblfish.net/counter") + val counterReq = + Request[IO](GET, counterUri, headers = Headers(`Cache-Control`(CacheDirective.`no-cache`()))) + + case class CacheInfo( + worldPeace: Int, + bbl: Int, + blogDir: Int, + counter: Int + ) + + def parseBody(body: String): CacheInfo = + val c1 = parseMap(body) + CacheInfo( + c1.getOrElse(worldPeace.path.toString, 0), + c1.getOrElse(bbl.path.toString, 0), + c1.getOrElse(bblBlogDir.path.toString, 0), + c1.getOrElse(counterUri.path.toString, 0) + ) + end InterpretedCacheMiddleTest class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: @@ -79,13 +99,17 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: enhance = _.putHeaders(Accept(MediaType.text.plain)) ) cc = stringCacheMiddleWare(WebTest.httpRoutes[IO].orNotFound) + rescounter0 <- cc.run(counterReq) respWP1 <- cc.run(Request[IO](GET, worldPeace)) respRoot <- cc.run(Request[IO](GET, bbl)) - rescounter1 <- cc.run(Request[IO](GET, counterUri)) + respBlog1 <- cc.run(Request[IO](HEAD, bblBlogDir)) + rescounter1 <- cc.run(counterReq) respWP2 <- cc.run(Request[IO](GET, worldPeace)) - rescounter2 <- cc.run(Request[IO](GET, counterUri)) + respBlog2 <- cc.run(Request[IO](GET, bblBlogDir)) + rescounter2 <- cc.run(counterReq) respRoot2 <- cc.run(Request[IO](GET, bbl)) - rescounter3 <- cc.run(Request[IO](GET, counterUri)) + respBlog3 <- cc.run(Request[IO](HEAD, bblBlogDir)) + rescounter3 <- cc.run(counterReq) cachedWorldPeace <- cache.lookup(worldPeace) closestDir1 <- cache.findClosest(bblBlogVirgin)(_.isDefined) wpAcrLink = cachedWorldPeace.flatMap { ci => @@ -108,7 +132,20 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: closestDir2 <- cache.findClosest(bblBlogVirgin)(_.isDefined) cachedBlogDirAcl <- cc.run(Request[IO](Method.GET, blogDirAcr.get)) cachedBlogDir2 <- cache.lookup(blogDirUrl) + respBlog4 <- cc.run( + Request[IO]( + GET, + bblBlogDir, + headers = Headers(Authorization(Credentials.Token(AuthScheme.Bearer, "abc"))) + ) + ) + rescounter4 <- cc.run(counterReq) yield + // this tells us that we start with the server having all counters at 0 + rescounter0.body.map { body => + assertEquals(parseBody(body), CacheInfo(0, 0, 0, 1), body) + }.getOrElse(fail("no body for 2nd req to " + counterUri)) + assertEquals(respWP1.status, Status.Ok) assertEquals( removeDate(respWP1.headers), @@ -127,13 +164,20 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: respRoot.body, Some(WebTest.rootTtl) ) + assertEquals(respBlog1.status, Status.Unauthorized) + assertEquals( + removeDate(respBlog1.headers), + WebTest.bblBlogRootHeader + ) + assertEquals( + respBlog1.body, + Some(""), + "the body on the access controlled resource is empty, as the result is unauthorized" + ) assertEquals(rescounter1.status, Status.Ok) // this tells us that we hit the cache once rescounter1.body.map { body => - val c1 = parseMap(body) - assertEquals(c1(worldPeace.path.toString), 1) - assertEquals(c1(bbl.path.toString), 1) - assertEquals(c1(counterUri.path.toString), 1) + assertEquals(parseBody(body), CacheInfo(1, 1, 1, 2), body) }.getOrElse(fail("no body")) assertEquals(respWP2.status, Status.Ok) @@ -141,14 +185,27 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: removeDate(respWP1.headers), WebTest.headers("world-at-peace", Some("/people/henry/blog/"), MediaType.text.plain) ) - assertEquals(rescounter2.status, Status.Ok) - // this tells us that we got all the info directly from the cache + assertEquals(respBlog2.status, Status.Unauthorized) + assertEquals( + removeDate(respBlog2.headers), + WebTest.bblBlogRootHeader + ) + assertEquals( + respBlog2.body, + Some(""), + "the body on the access controlled resource is empty, as the result is unauthorized" + ) + + // a HEAD after a GET should also get info directly from the cache + assertEquals(rescounter2.status, Status.Ok) rescounter2.body.map { body => - val c1 = parseMap(body) - assertEquals(c1(worldPeace.path.toString), 1) - assertEquals(c1(bbl.path.toString), 1) - assertEquals(c1(counterUri.path.toString), 1) + assertEquals( + parseBody(body), + CacheInfo(1, 1, 2, 3), + "A GET after a head on the blogDir requires a new request to the server, though arguably\n" + + " after a 401 the result should be the same unless and Authorization header is present." + ) }.getOrElse(fail("no body for 2nd req to " + counterUri)) assertEquals(respRoot2.status, Status.Ok) @@ -156,45 +213,61 @@ class InterpretedCacheMiddleTest extends munit.CatsEffectSuite: respRoot2.body, Some(WebTest.rootTtl) ) + assertEquals(respBlog3.status, Status.Unauthorized) + assertEquals( + removeDate(respBlog3.headers), + WebTest.bblBlogRootHeader + ) + assertEquals( + respBlog3.body, + None, + "A HEAD Request strips the body" + ) + // this tells us that we got all the info directly from the cache rescounter3.body.map { body => - val c1 = parseMap(body) - assertEquals(c1(worldPeace.path.toString), 1) - assertEquals(c1(bbl.path.toString), 1) - assertEquals(c1(counterUri.path.toString), 1) + assertEquals(parseBody(body), CacheInfo(1, 1, 2, 4), body) }.getOrElse(fail("no body for 3rd req to " + counterUri)) // test direct access to cache assertEquals(cachedWorldPeace.get.response.status, Status.Ok) assertEquals(cachedWorldPeace.get.response.body, Some(WebTest.bblWorldAtPeace)) - assertEquals(cacheBlogDir.status, Status.NotFound) + assertEquals(cacheBlogDir.status, Status.Unauthorized) assertEquals( removeDate(cacheBlogDir.headers), - Headers( - Link( - LinkValue(Uri(path = Uri.Path(Vector(Uri.Path.Segment(".acr")))), rel = Some("acl")) - ) - ) + WebTest.bblBlogRootHeader ) - // the closest dir is the root because we have not cached the blog dir - assertEquals(closestDir1.map(_.response.body), Some(Some(WebTest.rootTtl))) + // the closest dir to the virgin blog is the bblBlogDir because we cached it above + assertEquals(closestDir1.map(_.response.status), Some(Status.Unauthorized)) + assertEquals(closestDir1.map(_.response.body), Some(Some(""))) // now the blog dir is cached - assertEquals(closestDir2.map(_.response.body), Some(Some(WebTest.bblBlogRootContainer))) + assertEquals( + closestDir2.map(_.response.body), + Some(Some("")), + "the body is empty because it is a 401" + ) // we got the acl for the blog dir after following the link header of a 401 resource assertEquals(cachedBlogDirAcl.status, Status.Ok) assertEquals(cachedBlogDirAcl.body, Some(WebTest.bblBlogAcr)) - // the cache should have the same content as the response, even when returning a 404 + // the cache should have the same content as the response, even when returning a 401 // (ie, we did not have to make a new request to the web) - assertEquals(cachedBlogDir2.get.response.status, Status.NotFound) + assertEquals(cachedBlogDir2.get.response.status, Status.Unauthorized) assertEquals( removeDate(cachedBlogDir2.get.response.headers), - Headers( - Link( - LinkValue(Uri(path = Uri.Path(Vector(Uri.Path.Segment(".acr")))), rel = Some("acl")) - ) - ) + WebTest.bblBlogRootHeader + ) + + // now we make a get request with a bearer token, giving us a body + assertEquals(respBlog4.status, Status.Ok) + assertEquals( + respBlog4.body, + Some(WebTest.bblBlogRootContainer) ) + // so that bearer token request made a new connection to the server + rescounter4.body.map { body => + assertEquals(parseBody(body), CacheInfo(1, 1, 3, 5), body) + }.getOrElse(fail("no body for 4rth req to " + counterUri)) } diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/WebTest.scala b/cache/shared/src/test/scala/run/cosy/http/cache/WebTest.scala index ea9e4a4..24d48f9 100644 --- a/cache/shared/src/test/scala/run/cosy/http/cache/WebTest.scala +++ b/cache/shared/src/test/scala/run/cosy/http/cache/WebTest.scala @@ -32,6 +32,7 @@ import cats.FlatMap import cats.data.OptionT import cats.Monad import cats.effect.kernel.Clock +import io.chrisdavenport.mules.http4s.CachedResponse // import io.chrisdavenport.mules.http4s.CachedResponse.body object WebTest: @@ -66,6 +67,10 @@ object WebTest: // relative URLs val thisDoc = Uri.unsafeFromString("") val thisDir = Uri.unsafeFromString(".") + val bblBlogRootHeader = Headers( + Link(LinkValue(Uri(path = Uri.Path(Vector(Uri.Path.Segment(".acr")))), rel = Some("acl"))) + ) + val rootAcr = """ |@prefix wac: . |@prefix foaf: . @@ -116,11 +121,29 @@ object WebTest: | wac:default <.> . |""".stripMargin - val bblBlogRootContainer = "".stripMargin + val bblBlogRootContainer = """ + |@prefix ldp: . + | + |<> a ldp:BasicContainer; + | ldp:contains <2023/> . + |""".stripMargin val bblWorldAtPeace = "Hello World!" val bblBlogVirgin = "Play in three acts" + // we return a different response if there is an Authorization header (which we pretend is correct) + def bblBlogDir[F[_]](req: Request[F]): Response[F] = req.headers.get[Authorization] match + case Some(_) => Response[F]( + status = Status.Ok, + entity = Entity.strict(ByteVector(bblBlogRootContainer.getBytes)), + headers = bblBlogRootHeader ++ Headers(`Content-Type`(MediaType.text.turtle)) + ) + case None => Response[F]( + status = Status.Unauthorized, + entity = Entity.empty, + headers = bblBlogRootHeader + ) + def httpRoutes[F[_]: Monad: Clock](using AS: Async[F] ): HttpRoutes[F] = @@ -133,52 +156,48 @@ object WebTest: } }) >> OptionT.pure(req) } - val routes: Kleisli[OptionT[F, *], Request[F], Response[F]] = HttpRoutes.of[F] { - case GET -> Root => AS.pure( - Response[F]( - status = Status.Ok, - entity = Entity.strict(ByteVector(rootTtl.getBytes())), - headers = headers("/") - ) - ) - case GET -> Root / ".acr" => AS.pure( - Response[F]( - status = Status.Ok, - entity = Entity.strict(ByteVector(rootAcr.getBytes())), - headers = headers("/") - ) - ) - case GET -> Root / "people" / "henry" / "card" => AS.pure( - Response( - status = Status.Ok, - entity = Entity.strict(ByteVector(bblCardTtl.getBytes())), - headers = headers("card", Some("/")) - ) - ) - case GET -> Root / "people" / "henry" / "blog" / "2023" / "05" / "18" / "birth" => OK[F]( - bblBlogVirgin, - headers("birth", Some("/people/henry/blog/"), MediaType.text.plain) - ) - case GET -> Root / "people" / "henry" / "blog" / "" => // <- is this "ends with slash"? - Async[F].pure( - Response[F]( - status = Status.NotFound, - entity = Entity.empty, - headers = Headers(Link(LinkValue(Uri(path = Uri.Path(Vector(Uri.Path.Segment(".acr")))), rel = Some("acl")))) - ) - ) - case GET -> Root / "people" / "henry" / "blog" / "2023" / "04" / "01" / "world-at-peace" => - OK[F]( - bblWorldAtPeace, - headers("world-at-peace", Some("/people/henry/blog/"), MediaType.text.plain) - ) - case GET -> Root / "people" / "henry" / "blog" / ".acr" => OK[F]( - bblBlogAcr, - headers("") - ) - case GET -> Root / "counter" => - val cntStr = counter.get().toList.map { (p, i) => p.toString + " -> " + i }.mkString("\n") - OK[F](cntStr, headers("/counter", Some("/"), MediaType.text.plain)) + val routes: Kleisli[OptionT[F, *], Request[F], Response[F]] = HttpRoutes.of[F] { req => + req match + case GET -> Root => AS.pure( + Response[F]( + status = Status.Ok, + entity = Entity.strict(ByteVector(rootTtl.getBytes())), + headers = headers("/") + ) + ) + case GET -> Root / ".acr" => AS.pure( + Response[F]( + status = Status.Ok, + entity = Entity.strict(ByteVector(rootAcr.getBytes())), + headers = headers("/") + ) + ) + case GET -> Root / "people" / "henry" / "card" => AS.pure( + Response( + status = Status.Ok, + entity = Entity.strict(ByteVector(bblCardTtl.getBytes())), + headers = headers("card", Some("/")) + ) + ) + case GET -> Root / "people" / "henry" / "blog" / "2023" / "05" / "18" / "birth" => OK[F]( + bblBlogVirgin, + headers("birth", Some("/people/henry/blog/"), MediaType.text.plain) + ) + case HEAD -> Root / "people" / "henry" / "blog" / "" => Async[F].pure(bblBlogDir(req).copy(entity = Entity.empty)) + case GET -> Root / "people" / "henry" / "blog" / "" => Async[F].pure(bblBlogDir(req)) + case GET -> Root / "people" / "henry" / "blog" / "2023" / "04" / "01" / "world-at-peace" => + OK[F]( + bblWorldAtPeace, + headers("world-at-peace", Some("/people/henry/blog/"), MediaType.text.plain) + ) + case GET -> Root / "people" / "henry" / "blog" / ".acr" => OK[F]( + bblBlogAcr, + headers("") + ) + case GET -> Root / "counter" => + val cntStr = counter.get().toList.map { (p, i) => p.toString + " -> " + i } + .mkString("\n") + OK[F](cntStr, headers("/counter", Some("/"), MediaType.text.plain)) } val addTime: Kleisli[OptionT[F, *], Response[F], Response[F]] = Kleisli[OptionT[F, *], Response[F], Response[F]] { resp => From 34e7f0b713650d8b529551a15873729dc0851566 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Fri, 26 May 2023 19:05:12 +0200 Subject: [PATCH 35/42] move chrisdavenport's Cache.scala into the project --- build.sbt | 1 - .../scala/io/chrisdavenport/mules/Cache.scala | 135 ++++++++++++++++++ .../cache/InterpretedCacheMiddleware.scala | 1 + .../cache/{Cache.scala => TreeDirCache.scala} | 0 .../scala/run/cosy/http/cache/CacheTest.scala | 1 + .../cache/InterpretedCacheMiddleTest.scala | 1 + 6 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 cache/shared/src/main/scala/io/chrisdavenport/mules/Cache.scala rename cache/shared/src/main/scala/run/cosy/http/cache/{Cache.scala => TreeDirCache.scala} (100%) diff --git a/build.sbt b/build.sbt index de8188e..a416674 100644 --- a/build.sbt +++ b/build.sbt @@ -123,7 +123,6 @@ lazy val cache = crossProject(JVMPlatform).crossType(CrossType.Full).in(file("ca cats.free.value, http4s.core.value, http4s.client.value, - mules.core.value ), libraryDependencies ++= Seq( // munit.value % Test, diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/Cache.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/Cache.scala new file mode 100644 index 0000000..25845bf --- /dev/null +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/Cache.scala @@ -0,0 +1,135 @@ +package io.chrisdavenport.mules + +import cats._ +import cats.syntax.all._ + +trait Lookup[F[_], K, V]{ + def lookup(k: K): F[Option[V]] +} + +object Lookup { + + def mapValues[F[_]: Functor, K, A, B](l: Lookup[F, K, A])(f: A => B): Lookup[F, K, B] = + new Lookup[F, K, B]{ + def lookup(k: K): F[Option[B]] = l.lookup(k).map(_.map(f)) + } + + def contramapKeys[F[_], K, B, A](l: Lookup[F, K, A])(f: B => K): Lookup[F, B, A] = + new Lookup[F, B, A]{ + def lookup(k: B): F[Option[A]] = l.lookup(f(k)) + } + + def evalMap[F[_]: Monad, K, A, B](l: Lookup[F, K, A])(f: A => F[B]): Lookup[F, K, B] = + new Lookup[F, K, B]{ + def lookup(k: K): F[Option[B]] = l.lookup(k).flatMap(_.traverse(f)) + } + + def mapK[F[_], G[_], K, V](l: Lookup[F, K, V])(fk: F ~> G): Lookup[G, K, V] = + new Lookup[G, K, V]{ + def lookup(k: K): G[Option[V]] = fk(l.lookup(k)) + } +} + +trait Get[F[_], K, V]{ + def get(k: K): F[V] +} + +object Get { + + def mapValues[F[_]: Functor, K, A, B](l: Get[F, K, A])(f: A => B): Get[F, K, B] = + new Get[F, K, B]{ + def get(k: K): F[B] = l.get(k).map(f) + } + + def contramapKeys[F[_], K, B, A](l: Get[F, K, A])(g: B => K): Get[F, B, A] = + new Get[F, B, A]{ + def get(k: B): F[A] = l.get(g(k)) + } + + def evalMap[F[_]: Monad, K, A, B](l: Get[F, K, A])(f: A => F[B]): Get[F, K, B] = + new Get[F, K, B]{ + def get(k: K): F[B] = l.get(k).flatMap(f) + } + + def mapK[F[_], G[_], K, V](g: Get[F, K, V])(fk: F ~> G): Get[G, K, V] = + new Get[G, K, V]{ + def get(k: K): G[V] = fk(g.get(k)) + } + +} + +trait Insert[F[_], K, V]{ + def insert(k: K, v: V): F[Unit] +} + +object Insert { + + def contramapValues[F[_], K, A, B](i: Insert[F, K, A])(g: B => A): Insert[F, K, B] = + new Insert[F, K, B]{ + def insert(k: K, v: B): F[Unit] = i.insert(k, g(v)) + } + + def contramapKeys[F[_], A, B, V](i: Insert[F, A, V])(g: B => A): Insert[F, B, V] = + new Insert[F, B, V]{ + def insert(k: B, v: V): F[Unit] = i.insert(g(k), v) + } + + def mapK[F[_], G[_], K, V](i: Insert[F, K, V])(fk: F ~> G): Insert[G, K, V] = + new Insert[G, K, V]{ + def insert(k: K, v: V): G[Unit] = fk(i.insert(k, v)) + } + +} + +trait Delete[F[_], K]{ + def delete(k: K): F[Unit] +} + +object Delete { + def contramap[F[_], A, B](d: Delete[F, A])(g: B => A): Delete[F, B] = + new Delete[F, B]{ + def delete(k: B) = d.delete(g(k)) + } + + def mapK[F[_], G[_], K](d: Delete[F, K])(fk: F ~> G): Delete[G, K] = + new Delete[G, K]{ + def delete(k: K): G[Unit] = fk(d.delete(k)) + } +} + +trait Cache[F[_], K, V] + extends Lookup[F, K, V] + with Insert[F, K, V] + with Delete[F, K] + +object Cache { + def imapValues[F[_]: Functor, K, A, B](cache: Cache[F, K, A])(f: A => B, g: B => A): Cache[F, K, B] = + new Cache[F, K, B]{ + def lookup(k: K): F[Option[B]] = cache.lookup(k).map(_.map(f)) + def insert(k: K, v: B): F[Unit] = cache.insert(k, g(v)) + def delete(k: K): F[Unit] = cache.delete(k) + } + + def contramapKeys[F[_], K1, K2, V](c: Cache[F, K1, V])(g: K2 => K1): Cache[F, K2, V] = + new Cache[F, K2, V] { + def lookup(k: K2): F[Option[V]] = c.lookup(g(k)) + def insert(k: K2, v: V): F[Unit] = c.insert(g(k), v) + def delete(k: K2): F[Unit] = c.delete(g(k)) + } + + def evalIMap[F[_]: Monad, K, V1, V2]( + c: Cache[F, K, V1] + )(f: V1 => F[V2], g: V2 => V1): Cache[F, K, V2] = + new Cache[F, K, V2] { + def lookup(k: K): F[Option[V2]] = c.lookup(k).flatMap(_.traverse(f)) + def insert(k: K, v: V2): F[Unit] = c.insert(k, g(v)) + def delete(k: K): F[Unit] = c.delete(k) + } + + def mapK[F[_], G[_], K, V](cache: Cache[F, K, V])(fk: F ~> G): Cache[G, K, V] = + new Cache[G, K, V]{ + def lookup(k: K): G[Option[V]] = fk(cache.lookup(k)) + def insert(k: K, v: V): G[Unit] = fk(cache.insert(k, v)) + def delete(k: K): G[Unit] = fk(cache.delete(k)) + } +} \ No newline at end of file diff --git a/cache/shared/src/main/scala/run/cosy/http/cache/InterpretedCacheMiddleware.scala b/cache/shared/src/main/scala/run/cosy/http/cache/InterpretedCacheMiddleware.scala index 9cc54b2..d72af7c 100644 --- a/cache/shared/src/main/scala/run/cosy/http/cache/InterpretedCacheMiddleware.scala +++ b/cache/shared/src/main/scala/run/cosy/http/cache/InterpretedCacheMiddleware.scala @@ -23,6 +23,7 @@ import io.chrisdavenport.mules.http4s.{CacheItem, CacheType, CachedResponse} import org.http4s.* import org.http4s.client.Client import cats.arrow.FunctionK +import run.cosy.http.cache.TreeDirCache /** Interpreted Cache don't just cache the resource but the interpretation of that resource, e.g. * the parsed JSON or XML, or an RDF Graph or Quads diff --git a/cache/shared/src/main/scala/run/cosy/http/cache/Cache.scala b/cache/shared/src/main/scala/run/cosy/http/cache/TreeDirCache.scala similarity index 100% rename from cache/shared/src/main/scala/run/cosy/http/cache/Cache.scala rename to cache/shared/src/main/scala/run/cosy/http/cache/TreeDirCache.scala diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/CacheTest.scala b/cache/shared/src/test/scala/run/cosy/http/cache/CacheTest.scala index 3ce8165..34de046 100644 --- a/cache/shared/src/test/scala/run/cosy/http/cache/CacheTest.scala +++ b/cache/shared/src/test/scala/run/cosy/http/cache/CacheTest.scala @@ -20,6 +20,7 @@ import cats.MonadError import cats.effect.{IO, Ref, SyncIO} import munit.CatsEffectSuite import org.http4s.Uri +import run.cosy.http.cache.{TreeDirCache, ServerNotFound} class CacheTest extends CatsEffectSuite: import TreeDirCache.* diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala b/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala index c66e161..95ec6a5 100644 --- a/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala +++ b/cache/shared/src/test/scala/run/cosy/http/cache/InterpretedCacheMiddleTest.scala @@ -17,6 +17,7 @@ import io.chrisdavenport.mules.http4s.CacheItem import org.typelevel.ci.CIStringSyntax import cats.data.NonEmptyList import io.chrisdavenport.mules.http4s.internal.Caching +import run.cosy.http.cache.TreeDirCache object InterpretedCacheMiddleTest: def bytesToString(bytes: Vector[Byte]): String = bytes.map(_.toChar).mkString From b35d8c6413633eed8b27625f1d071765c49e6095 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Fri, 2 Jun 2023 20:22:44 +0200 Subject: [PATCH 36/42] see https://discord.com/channels/632277896739946517/632277897448652844/1114252901502697542 --- .scalafmt.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/.scalafmt.conf b/.scalafmt.conf index b7254eb..701a6c7 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -15,6 +15,7 @@ rewrite.scala3 { convertToNewSyntax = true removeOptionalBraces = yes } +runner.dialectOverride.allowQuestionMarkAsTypeWildcard = false newlines { selectChains = fold beforeMultiline = fold From 31de0141481d11420d2eb851456436b5ab99bde6 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Fri, 2 Jun 2023 20:56:20 +0200 Subject: [PATCH 37/42] pre-sigining using cache (works but uses web) --- build.sbt | 2 +- .../scala/io/chrisdavenport/mules/Cache.scala | 8 + .../run/cosy/http/cache/TreeDirCache.scala | 29 +- .../scala/run/cosy/web/util/UrlUtil.scala | 27 ++ .../scala/run/cosy/web/util/UrlUtilTest.scala | 23 ++ project/Dependencies.scala | 2 +- .../net/bblfish/wallet/BasicAuthWallet.scala | 292 +++++++++++++----- 7 files changed, 286 insertions(+), 97 deletions(-) create mode 100644 ioExt4s/shared/src/test/scala/run/cosy/web/util/UrlUtilTest.scala diff --git a/build.sbt b/build.sbt index a416674..48d614c 100644 --- a/build.sbt +++ b/build.sbt @@ -122,7 +122,7 @@ lazy val cache = crossProject(JVMPlatform).crossType(CrossType.Full).in(file("ca cats.core.value, cats.free.value, http4s.core.value, - http4s.client.value, + http4s.client.value ), libraryDependencies ++= Seq( // munit.value % Test, diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/Cache.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/Cache.scala index 25845bf..7cbc7c3 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/Cache.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/Cache.scala @@ -97,6 +97,14 @@ object Delete { } } + +/** Local Search function into the cache */ +trait LocalSearch[F[_], K, V]: + /* find closest value from key going down the URL hierarchy that satisifies predicate. + * the domain is an Option[K], so the function can decided what to do if a node does not exist + */ + def findClosest(k: K)(predicate: Option[K] => Boolean): F[Option[V]] + trait Cache[F[_], K, V] extends Lookup[F, K, V] with Insert[F, K, V] diff --git a/cache/shared/src/main/scala/run/cosy/http/cache/TreeDirCache.scala b/cache/shared/src/main/scala/run/cosy/http/cache/TreeDirCache.scala index 63f9cf3..e7aacd8 100644 --- a/cache/shared/src/main/scala/run/cosy/http/cache/TreeDirCache.scala +++ b/cache/shared/src/main/scala/run/cosy/http/cache/TreeDirCache.scala @@ -19,7 +19,7 @@ package run.cosy.http.cache import cats.effect.kernel.{Ref, Sync} import cats.syntax.all.* import cats.{FlatMap, MonadError} -import io.chrisdavenport.mules.Cache +import io.chrisdavenport.mules.{Cache,LocalSearch} import org.http4s.Uri import run.cosy.http.cache.DirTree.* import run.cosy.http.cache.TreeDirCache.WebCache @@ -94,16 +94,17 @@ case class TreeDirCache[F[_], X]( val (path, v) = tree.find(k.path.segments) if path.isEmpty then v else None - /** find the closest node matching `select` going backwards from the closest node we have leading - * to path. So if we want but we have - * and but only that content at the latter - * resource matches, then we will get that. - */ - def findClosest(k: Uri)(matcher: Option[X] => Boolean): F[Option[X]] = - for - scheme <- F.fromOption(k.scheme, IncompleteServiceInfo(k)) - auth <- F.fromOption(k.authority, IncompleteServiceInfo(k)) - webCache <- cacheRef.get - server = (scheme, auth) - tree <- F.fromOption(webCache.get(server), ServerNotFound(k)) - yield tree.findClosest(k.path.segments)(matcher).flatten + // /** find the closest node matching `select` going backwards from the closest node we have leading + // * to path. So if we want but we have + // * and but only that content at the latter + // * resource matches, then we will get that. + // */ + // def findClosest(k: Uri)(predicate: Option[X] => Boolean): F[Option[X]] = + // for + // scheme <- F.fromOption(k.scheme, IncompleteServiceInfo(k)) + // auth <- F.fromOption(k.authority, IncompleteServiceInfo(k)) + // webCache <- cacheRef.get + // server = (scheme, auth) + // tree <- F.fromOption(webCache.get(server), ServerNotFound(k)) + // yield tree.findClosest(k.path.segments)(predicate).flatten + diff --git a/ioExt4s/shared/src/main/scala/run/cosy/web/util/UrlUtil.scala b/ioExt4s/shared/src/main/scala/run/cosy/web/util/UrlUtil.scala index f7b5bb3..42deb1f 100644 --- a/ioExt4s/shared/src/main/scala/run/cosy/web/util/UrlUtil.scala +++ b/ioExt4s/shared/src/main/scala/run/cosy/web/util/UrlUtil.scala @@ -29,6 +29,33 @@ object UrlUtil: extension (llUri: ll.Url) def toh4: org.http4s.Uri = llUrltoHttp4s(llUri) + extension (llUrl: ll.AbsoluteUrl) + /** the parent url is the container containing this url most directly. Always ends in slash. + * The root has itself as container + */ + def parent: Option[ll.AbsoluteUrl] = + given ll.config.UriConfig = ll.config.UriConfig.default + val parentPathOpt = llUrl.path match + case ap: ll.AbsolutePath => ap.parent + case _ => None + parentPathOpt.map(x => + llUrl.copy( + path = x, + query = ll.QueryString.empty, + fragment = None + ) + ) + + extension (absPath: ll.AbsolutePath) + def parent: Option[ll.AbsolutePath] = + val dropedSlash = + if absPath.parts.last == "" + then absPath.parts.dropRight(1) + else absPath.parts + if dropedSlash.isEmpty + then None + else Some(new ll.AbsolutePath(dropedSlash.updated(dropedSlash.length - 1, ""))) + extension [R <: RDF](uri: RDF.URI[R])(using ops: Ops[R]) def toLL: Try[ll.Uri] = import ops.{*, given} diff --git a/ioExt4s/shared/src/test/scala/run/cosy/web/util/UrlUtilTest.scala b/ioExt4s/shared/src/test/scala/run/cosy/web/util/UrlUtilTest.scala new file mode 100644 index 0000000..ade21b6 --- /dev/null +++ b/ioExt4s/shared/src/test/scala/run/cosy/web/util/UrlUtilTest.scala @@ -0,0 +1,23 @@ +package run.cosy.web.util + +import io.lemonlabs.uri as ll +import run.cosy.web.util.UrlUtil.* + +class UrlUtilTest extends munit.FunSuite { + def p(s: String) = ll.AbsoluteUrl.parseOption(s) + val bblRoot = p("https://bblfish.net/") + val bblCard = p("https://bblfish.net/people/henry/card#me") + val henry = p("https://bblfish.net/people/henry/") + val ppl = p("https://bblfish.net/people/") + val bbldoc = p("https://bblfish.net") + + test("parent test") { + assert(henry.isDefined) + assertEquals(bblCard.flatMap(_.parent), henry) + assert(henry.isDefined) + assertEquals(henry.flatMap(_.parent), ppl) + assertEquals(ppl.flatMap(_.parent), bblRoot) + assertEquals(bblRoot.flatMap(_.parent), None) + } + +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index cdd10bd..bb8b51e 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -3,7 +3,7 @@ import sbt.{Def, *} object Dependencies { object Ver { - val scala = "3.2.2" + val scala = "3.3.0" val http4s = "1.0.0-M39" val banana = "0.9-c996591-SNAPSHOT" val bobcats = "0.3-3236e64-SNAPSHOT" diff --git a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala index 7dc9988..0b4eaed 100644 --- a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala +++ b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala @@ -65,6 +65,8 @@ import io.lemonlabs.uri.AbsoluteUrl import io.lemonlabs.uri.config.UriConfig import BasicWallet.* import org.http4s.Method +import org.http4s.Method.{GET, HEAD} +import org.http4s.dsl.request class BasicId(val username: String, val password: String) @@ -103,6 +105,11 @@ object BasicWallet: val defaultACOpt = Some(defaultAC) val ldpContains = "http://www.w3.org/ns/ldp#contains" + /** The type of relations that can be found in the Link header, Left is reverse and Right is + * forward, and the link is to the absoluteUrl from the document + */ + type Rel = (Either[String, String], ll.AbsoluteUrl) + /** extract the links from the headers as pairs of Eithers for rev or rel relations and the url * value, and absolutize the Url. Todo: we use ll.Uri here, because they have a clear type for * absolute urls, but this is a we really need a url abstraction that does this right @@ -110,7 +117,7 @@ object BasicWallet: def extractLinks( requestUrl: AbsoluteUrl, reponseHeaders: org.http4s.Headers - ): Seq[(Either[String, String], ll.AbsoluteUrl)] = reponseHeaders.get[Link] match + ): Seq[Rel] = reponseHeaders.get[Link] match case None => Seq() case Some(link) => link.values.toList.toSeq.collect { case LinkValue(uri, rel, rev, _, _) if rel.isDefined || rev.isDefined => @@ -151,6 +158,8 @@ class WalletTools[Rdf <: RDF](using ops: Ops[Rdf]): * todo: is the final URL that a redirect goes to the URL to use to absolutize a graph, or is * the request URL after going through the redirects? Because that is important to know how much * info is needed to be able to interpret the relative graph. + * + * Todo: also store graphs for other responses? (e.g. error responses?) */ def cachedRelGraphMiddleware[F[_]: Concurrent: Clock]( cache: TreeDirCache[F, CacheItem[RDF.rGraph[Rdf]]] @@ -160,15 +169,17 @@ class WalletTools[Rdf <: RDF](using ops: Ops[Rdf]): .client[F, RDF.rGraph[Rdf]]( cache, (response: h4s.Response[F]) => - if !response.status.isSuccess - then - Concurrent[F].pure(CachedResponse[RDF.rGraph[Rdf]]( - response.status, - response.httpVersion, - response.headers, - None - )) - else + if !response.status.isSuccess + then + Concurrent[F].pure( + CachedResponse[RDF.rGraph[Rdf]]( + response.status, + response.httpVersion, + response.headers, + None + ) + ) + else import rdfDecoders.allrdf response.as[RDF.rGraph[Rdf]].map { rG => CachedResponse[RDF.rGraph[Rdf]]( @@ -182,6 +193,7 @@ class WalletTools[Rdf <: RDF](using ops: Ops[Rdf]): enhance = _.withHeaders(rdfDecoders.allRdfAccept) ) + /** The requesturi is within the container */ def withinTry(requestUri: RDF.URI[Rdf], container: RDF.URI[Rdf]): Try[Boolean] = for case requ: ll.AbsoluteUrl <- requestUri.toLL @@ -292,49 +304,146 @@ class BasicWallet[F[_], Rdf <: RDF]( end basicChallenge // todo: do we need the return to be a Resource? Would F[Graph] be enough? - /** - * Fetches the ACL for the given uri, and if it is not found, tries to find the ACL for the - * linkedToContainer (not implemented yet). - */ + /** Fetches the ACL for the given uri, and if it is not found, tries to find the ACL for the + * linkedToContainer (not implemented yet). + */ def fetchAclGr( uri: AbsoluteUrl, - fallbackContainer: Option[ll.AbsoluteUrl] = None - ): Resource[F, RDF.Graph[Rdf]] = iClient.run(h4s.Request(uri = uri.toh4.withoutFragment)) - .flatMap { crG => - Resource.liftK( - if crG.status.isSuccess && crG.body.isDefined - then fc.pure(crG.body.get.resolveAgainst(uri)) - else fc.raiseError(Exception(s"Could not find ACL at $uri")) - ) - } - - def findAclForContainer(containerUrl: AbsoluteUrl): Resource[F, RDF.Graph[Rdf]] = - // todo: should be a HEAD request!!! - iClient.run(h4s.Request(Method.HEAD, uri = containerUrl.toh4.withoutFragment)).flatMap { crG => - // todo: potentially check for the right status codes... - // which ones would be correct? Clearly 201, 202, 404. But what others? - extractLinks(containerUrl, crG.headers).collectFirst { case (Right("acl"), uri) => - fetchAclGr(uri) + onlyCache: Boolean = false + ): Resource[F, RDF.Graph[Rdf]] = fetchRdf(uri, GET, onlyCache).flatMap { crG => + Resource.liftK( + if crG.status.isSuccess && crG.body.isDefined + then fc.pure(crG.body.get.resolveAgainst(uri)) + else fc.raiseError(Exception(s"Could not find ACL at $uri")) + ) + } + + def findAclForContainer( + containerUrl: AbsoluteUrl, + onlyCache: Boolean = false + ): Resource[F, RDF.Graph[Rdf]] = fetchRdf(containerUrl, HEAD, onlyCache).flatMap { crG => + // todo: potentially check for the right status codes... + // which ones would be correct? Clearly 201, 202, 404. But what others? + extractLinks(containerUrl, crG.headers) + .collectFirst { case (Right("acl"), uri: ll.AbsoluteUrl) => + fetchAclGr(uri.resolve(containerUrl).toAbsoluteUrl, onlyCache) }.getOrElse { - Resource.raiseError[F,RDF.Graph[Rdf],Throwable](Throwable(s"Could not find ACL for $containerUrl")) + Resource.raiseError[F, RDF.Graph[Rdf], Throwable]( + Throwable(s"Could not find ACL for $containerUrl") + ) } - - } - + } + + /** fetch rdf graph at Url using Method, adding an `only-if-cached` header if needed. */ + def fetchRdf( + url: ll.AbsoluteUrl, + method: h4s.Method = GET, + onlyCache: Boolean = false + ): Resource[F, CachedResponse[RDF.Graph[Rdf]]] = + import h4s.CacheDirective.`only-if-cached` + // we don't need the `Accept` headers as those are added by the cache middleware + val headers = + if onlyCache then h4s.Headers(h4s.headers.`Cache-Control`(`only-if-cached`)) + else h4s.Headers.empty + iClient.run(h4s.Request(method = method, uri = url.toh4.withoutFragment, headers = headers)) + .map(cachedRes => cachedRes.map(_.resolveAgainst(url))) + + /** we find ACL by going to the web */ def findAclFor( requestUrl: ll.AbsoluteUrl, - responseHeaders: org.http4s.Headers + responseHeaders: org.http4s.Headers, + depth: Int = 50 ): Resource[F, RDF.Graph[Rdf]] = - val links = extractLinks(requestUrl, responseHeaders).sortBy(x => relPriority(x._1)) - val default = links.collectFirst { - case (Right(`defaultAC`), uri) => findAclForContainer(uri) - case (Right("acl"), uri) => fetchAclGr(uri) + val onlyCache: Boolean = false + val links: Seq[Rel] = extractLinks(requestUrl, responseHeaders).sortBy(x => relPriority(x._1)) + val acl = links.collectFirst { + case (Right(`defaultAC`), uri) => findAclForContainer(uri, onlyCache) + case (Right("acl"), uri) => fetchAclGr(uri, onlyCache) + } + acl.getOrElse { + findContainerFor(requestUrl, links, onlyCache, depth) match + case None => Resource.raiseError[F, RDF.Graph[Rdf], Throwable]( + Exception(s"No useable link to find ACL from $requestUrl") + ) + case Some(url) => findAclFor(url, responseHeaders, depth - 1) } - default.getOrElse { - Resource.raiseError[F, RDF.Graph[Rdf],Throwable](Exception(s"No useable link to find ACL from $requestUrl")) - } end findAclFor + /** when looking in the cache we never assume that no info is bad info, we just continue looking, + * so we don't even assume we already have the data of the first request. (hence no headers) + * todo: see if we can combine this function and findAclFor into one function to avoid + * duplication + */ + def findCachedAclFor( + requestUrl: ll.AbsoluteUrl, + depth: Int = 50 + ): Resource[F, RDF.Graph[Rdf]] = + val onlyCache: Boolean = false + // we start by making a HEAD request, because we just want to find the link to the acl, + // if requestUrl is the acl, then it may link back to itself, giving us what we want, or it + // will link to nothing, in which case we won't notice which is a problem, in which case we won't notice which is a problem + // todo: have acls link back to themselves on the server as per https://github.com/solid/authorization-panel/issues/189 + // or find another way to allow the client to tell it is on an acl. + fetchRdf(requestUrl, HEAD, onlyCache).flatMap { crG => + val links = extractLinks(requestUrl, crG.headers) + // we signal an continuable failure by plaing it in the internal Option + val defaultOrRel: Resource[F, Option[RDF.Graph[Rdf]]] = + if onlyCache && crG.status == h4s.Status.GatewayTimeout + then // we retrun a resource with a None, meaning we can continue looking in the cache + Resource.liftK(fc.pure[Option[RDF.Graph[Rdf]]](None)) + else + for + optDeflt <- links.collectFirst { case (Right(`defaultAC`), uri) => + findAclForContainer(uri, onlyCache) + }.sequence + // if the defaultAC link above fails we fail otherwise + optEAcl <- optDeflt match + case Some(_) => optDeflt.map(Right[Throwable, RDF.Graph[Rdf]]) + .pure[Resource[F, _]] + case None => links.collectFirst { case (Right("acl"), uri) => + fetchAclGr(uri, onlyCache) + }.map(_.attempt).sequence + yield optEAcl.flatMap(_.toOption) + + // fetching acl directly can fail we should continue if it does + defaultOrRel.flatMap { + case Some(gr) => gr.pure[Resource[F, _]] + case None => findContainerFor(requestUrl, links, onlyCache, depth) match + case None => Resource.raiseError[F, RDF.Graph[Rdf], Throwable]( + Exception(s"No useable link to find ACL from $requestUrl") + ) + case Some(url) => findCachedAclFor(url, depth - 1) + } + } + end findCachedAclFor + + /** @param requestUrl + * the original request url + * @param links + * the links found in the response or previous request on requestUrl + * @param onlyCache + * if true, only the cache is searched + * @param depth + * the depth to search for ldp:contains reverse relations, after which URL hierarchy can be + * looked at if in onlyCache mode + * @return + * container for given URL looking at ldp:contains relation in header or if none is found in + * onlyCache mode looking at parent of requestUrl + */ + def findContainerFor( + requestUrl: ll.AbsoluteUrl, + links: Seq[Rel], + onlyCache: Boolean = false, + depth: Int + ): Option[ll.AbsoluteUrl] = links.collectFirst { + case (Right(`ldpContains`), uri) if depth >= 0 => uri.resolve(requestUrl).toAbsoluteUrl + }.orElse { + if onlyCache + then // we don't need to count depth on onlycache as Urls have a pragmetic size limit + requestUrl.parent + else None + } + /** given the original request and a response, return the correctly signed original request (test * for the HttpSig WWW-Authenticate header has been done before calling this function) */ @@ -346,46 +455,56 @@ class BasicWallet[F[_], Rdf <: RDF]( ): F[h4s.Request[F]] = import BasicWallet.* val result: Resource[F, h4s.Request[F]] = findAclFor(lastReqUrl, response.headers) - .flatMap { (aclGr: RDF.Graph[Rdf]) => - import io.lemonlabs.uri.config.UriConfig - val reqRes = ops.URI(lastReqUrl.copy(fragment = None)(UriConfig.default)) - val keyNodes: Iterator[St.Subject[Rdf]] = - for - agentNode <- findAgents(aclGr, reqRes, originalRequest.method) - controllerTriple <- aclGr.find(*, sec.controller, agentNode) - yield controllerTriple.subj - - import run.cosy.http4s.Http4sTp.given - val keys: Iterable[KeyData[F]] = keyNodes.collect { case u: RDF.URI[Rdf] => - keyIdDB.find(kid => kid.keyIdAtt.value.asciiStr == u.value).toList - }.flatten.to(Iterable) - - val x: F[h4s.Request[F]] = - for - keydt <- fc.fromOption[KeyData[F]]( - keys.headOption, - Exception( - s"none of our keys fit the ACL for resource $lastReqUrl accessed in " + - s"${originalRequest.method} matches the rules in graph { $aclGr } " - ) - ) - signingFn <- keydt.signer - now <- clock.realTime // <- todo, add clock time caching perhaps - signedReq <- MessageSignature.withSigInput[F, H4]( - originalRequest.asInstanceOf[Http.Request[H4]], - Rfc8941.Token("sig1"), - keydt.mkSigInput(now), - signingFn - ) - yield - val res = run.cosy.http4s.Http4sTp.hOps - .addHeader[Http.Request[H4]](signedReq)("Authorization", "HttpSig proof=sig1") - h4ReqToHttpReq(res) - Resource.liftK(x) - } + .flatMap(signRequest(originalRequest, lastReqUrl)) result.use(fc.pure) end httpSigChallenge + /** sign the request with the first key that matches the rules in the aclGraph. (We pass the requestUrl + * too to avoid recaculating it from the request...) + */ + def signRequest( + originalRequest: h4s.Request[F], + originalRequestUrl: ll.AbsoluteUrl + )( + aclGr: RDF.Graph[Rdf] + ): Resource[F, h4s.Request[F]] = + import io.lemonlabs.uri.config.UriConfig + val reqRes = ops.URI(originalRequestUrl.copy(fragment = None)(UriConfig.default)) + val keyNodes: Iterator[St.Subject[Rdf]] = + for + agentNode <- findAgents(aclGr, reqRes, originalRequest.method) + controllerTriple <- aclGr.find(*, sec.controller, agentNode) + yield controllerTriple.subj + + import run.cosy.http4s.Http4sTp.given + val keys: Iterable[KeyData[F]] = keyNodes.collect { case u: RDF.URI[Rdf] => + keyIdDB.find(kid => kid.keyIdAtt.value.asciiStr == u.value).toList + }.flatten.to(Iterable) + + val x: F[h4s.Request[F]] = + for + keydt <- fc.fromOption[KeyData[F]]( + keys.headOption, + Exception( + s"none of our keys fit the ACL for resource $originalRequestUrl accessed in " + + s"${originalRequest.method} matches the rules in graph { $aclGr } " + ) + ) + signingFn <- keydt.signer + now <- clock.realTime // <- todo, add clock time caching perhaps + signedReq <- MessageSignature.withSigInput[F, H4]( + originalRequest.asInstanceOf[Http.Request[H4]], + Rfc8941.Token("sig1"), + keydt.mkSigInput(now), + signingFn + ) + yield + val res = run.cosy.http4s.Http4sTp.hOps + .addHeader[Http.Request[H4]](signedReq)("Authorization", "HttpSig proof=sig1") + h4ReqToHttpReq(res) + Resource.liftK(x) + end signRequest + /** This is different from middleware such as FollowRedirects, as that essentially continues the * request. Here we need to stop the request and make new ones to find the access control rules * for the given resource. (that could just be a BasicAuth request for a password, or a more @@ -416,6 +535,17 @@ class BasicWallet[F[_], Rdf <: RDF]( case _ => ??? // fail end sign - override def signFromDB(req: h4s.Request[F]): F[h4s.Request[F]] = fc.point(req) + override def signFromDB(req: h4s.Request[F]): F[h4s.Request[F]] = + // todo: the DB needs to keep track of what WWW-Authenticate methods the server allows. + // These will be difficult to find in the headers, as the 401 in which they appeared may be + // somewhere completely different. + // I will assume that the server can do HTTP-Sig for the moment + // todo: fix!!! + // todo: as we endup with F[Request] do we need Resources everywhere here? + findCachedAclFor(req.uri.toLL.toAbsoluteUrl, 100).flatMap { + signRequest(req, req.uri.toLL.toAbsoluteUrl) + }.use(fc.pure).map { req => + println(s"===> signed request ${req}uri} with headers ${req.headers}"); req + } end BasicWallet From 7a3094e663f1b61b391355b4103fd45fa1cb6ab1 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Sat, 3 Jun 2023 15:10:03 +0200 Subject: [PATCH 38/42] Client now requires minimal over the wire http requests. --- .../scala/net/bblfish/app/auth/AuthNClient.scala | 13 +++++++++---- project/build.properties | 2 +- .../src/main/scala/net/bblfish/app/Wallet.scala | 4 ++-- .../scala/net/bblfish/wallet/BasicAuthWallet.scala | 10 ++++------ 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/authn/shared/src/main/scala/net/bblfish/app/auth/AuthNClient.scala b/authn/shared/src/main/scala/net/bblfish/app/auth/AuthNClient.scala index 7c3fe3d..f3ef8a4 100644 --- a/authn/shared/src/main/scala/net/bblfish/app/auth/AuthNClient.scala +++ b/authn/shared/src/main/scala/net/bblfish/app/auth/AuthNClient.scala @@ -34,7 +34,11 @@ import org.http4s.{BasicCredentials, Header, Request, Response, Status} import scala.util.{Failure, Success, Try} /** Client Authentication is a middleware that transforms a Client into a new Client that can use a - * Wallet to have requests signed. + * Wallet to have requests signed. It will try to sign a request + * 1. before it is sent using information it has available from the local cache on the server. So + * it will try to find an relevant ACL that it can use to determine if it can sign something + * 1. if the server returns a 401 it will use the response info to fetch the ACL rules and if it + * can, sign the request */ object AuthNClient: def apply[F[_]: Concurrent](wallet: Wallet[F])( @@ -60,9 +64,10 @@ object AuthNClient: // Not 100% sure this is so much needed here... Hotswap.create[F, Response[F]].flatMap { hotswap => Resource.eval( - wallet.signFromDB(req).flatMap { possiblySignedReq => - authLoop(possiblySignedReq, 0, hotswap) - } + wallet.signFromDB(req).map { + case Right(signedReq) => signedReq + case Left(_) => req + }.flatMap(req => authLoop(req, 0, hotswap)) ) } } diff --git a/project/build.properties b/project/build.properties index 72413de..40b3b8e 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.8.3 +sbt.version=1.9.0 diff --git a/wallet/shared/src/main/scala/net/bblfish/app/Wallet.scala b/wallet/shared/src/main/scala/net/bblfish/app/Wallet.scala index d86fc77..7d12b44 100644 --- a/wallet/shared/src/main/scala/net/bblfish/app/Wallet.scala +++ b/wallet/shared/src/main/scala/net/bblfish/app/Wallet.scala @@ -33,6 +33,6 @@ trait Wallet[F[_]]: /** previous requests to a server will return acls and methods that can be assumed to be valid * @param req - * @return + * @return a request with a signature if possible, otherwise an error that can be ignored */ - def signFromDB(req: Request[F]): F[Request[F]] + def signFromDB(req: Request[F]): F[Either[Throwable,Request[F]]] diff --git a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala index 0b4eaed..7264990 100644 --- a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala +++ b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala @@ -378,7 +378,7 @@ class BasicWallet[F[_], Rdf <: RDF]( requestUrl: ll.AbsoluteUrl, depth: Int = 50 ): Resource[F, RDF.Graph[Rdf]] = - val onlyCache: Boolean = false + val onlyCache: Boolean = true // we start by making a HEAD request, because we just want to find the link to the acl, // if requestUrl is the acl, then it may link back to itself, giving us what we want, or it // will link to nothing, in which case we won't notice which is a problem, in which case we won't notice which is a problem @@ -535,7 +535,7 @@ class BasicWallet[F[_], Rdf <: RDF]( case _ => ??? // fail end sign - override def signFromDB(req: h4s.Request[F]): F[h4s.Request[F]] = + override def signFromDB(req: h4s.Request[F]): F[Either[Throwable,h4s.Request[F]]] = // todo: the DB needs to keep track of what WWW-Authenticate methods the server allows. // These will be difficult to find in the headers, as the 401 in which they appeared may be // somewhere completely different. @@ -544,8 +544,6 @@ class BasicWallet[F[_], Rdf <: RDF]( // todo: as we endup with F[Request] do we need Resources everywhere here? findCachedAclFor(req.uri.toLL.toAbsoluteUrl, 100).flatMap { signRequest(req, req.uri.toLL.toAbsoluteUrl) - }.use(fc.pure).map { req => - println(s"===> signed request ${req}uri} with headers ${req.headers}"); req - } - + }.attempt.use(fc.pure) + end BasicWallet From 0960f23a1690290858eb39ed6417dba543240f0b Mon Sep 17 00:00:00 2001 From: Henry Story Date: Mon, 5 Jun 2023 07:41:59 +0200 Subject: [PATCH 39/42] scala-cli format . --- .../run/cosy/solid/app/http/Fetcher.scala | 12 +- .../cosy/solid/app/http/RDFMediaTypes.scala | 4 +- .../scala/run/cosy/solid/app/http/Web.scala | 4 +- app/src/main/scala/solidapp/Example.scala | 134 ++++----- .../solid/app/http/FetcherMunitTests.scala | 13 +- build.sbt | 3 +- .../scala/io/chrisdavenport/mules/Cache.scala | 180 +++++------- .../mules/http4s/CacheItem.scala | 3 +- .../mules/http4s/CachedResponse.scala | 2 +- .../mules/http4s/internal/Caching.scala | 6 +- .../scala/run/cosy/http/cache/DirTree.scala | 6 +- .../run/cosy/http/cache/TreeDirCache.scala | 3 +- .../scala/run/cosy/http/cache/WebTest.scala | 13 +- free/shared/net/bblfish/ldp/cmd/LDPCmd.scala | 61 ++-- .../run/cosy/ld/http4s/RDFDecoders.scala | 2 +- .../scala/run/cosy/web/util/UrlUtilTest.scala | 16 +- .../main/scala/n3js/n3/mod/BaseFormat.scala | 45 +-- n3js/src/main/scala/n3js/n3/mod/Lexer.scala | 83 +++--- .../main/scala/n3js/n3/mod/LexerOptions.scala | 74 +++-- n3js/src/main/scala/n3js/n3/mod/Logger.scala | 9 +- .../src/main/scala/n3js/n3/mod/MimeType.scala | 39 +-- n3js/src/main/scala/n3js/n3/mod/Parser.scala | 142 ++++----- .../scala/n3js/n3/mod/ParserOptions.scala | 95 +++--- n3js/src/main/scala/n3js/n3/mod/Star.scala | 25 +- n3js/src/main/scala/n3js/n3/mod/Token.scala | 78 +++-- n3js/src/main/scala/n3js/n3/mod/package.scala | 122 ++++---- n3js/src/main/scala/n3js/n3/n3Strings.scala | 273 ++++++++---------- n3js/src/main/scala/n3js/node/eventsMod.scala | 82 +++--- n3js/src/main/scala/n3js/std/ArrayLike.scala | 37 +-- n3js/src/main/scala/n3js/std/Iterable.scala | 9 +- n3js/src/main/scala/n3js/std/package.scala | 25 +- .../src/main/scala/n3js/std/stdBooleans.scala | 23 +- n3js/src/main/scala/n3js/std/stdStrings.scala | 13 +- .../scala/run/cosy/app/io/n3/N3Parser.scala | 96 +++--- .../solid/app/io/http/N3ParserTests.scala | 92 +++--- .../main/scala/scripts/AnHttpSigClient.scala | 2 +- .../shared/src/main/scala/scripts/PemToJWT.sc | 1 - .../main/scala/net/bblfish/app/Wallet.scala | 5 +- .../net/bblfish/wallet/BasicAuthWallet.scala | 14 +- 39 files changed, 881 insertions(+), 965 deletions(-) diff --git a/app/src/main/scala/run/cosy/solid/app/http/Fetcher.scala b/app/src/main/scala/run/cosy/solid/app/http/Fetcher.scala index 77ab40f..c908fd6 100644 --- a/app/src/main/scala/run/cosy/solid/app/http/Fetcher.scala +++ b/app/src/main/scala/run/cosy/solid/app/http/Fetcher.scala @@ -10,7 +10,6 @@ import org.http4s.implicits.* import org.http4s.{ParseResult, QValue, Request, Uri, client} import run.cosy.app.io.n3.N3Parser - import java.nio.charset.Charset import scala.scalajs.js.annotation.{JSExport, JSExportTopLevel} @@ -18,13 +17,8 @@ import org.w3.banana.RDF import org.w3.banana.RDF.* import org.w3.banana.Ops -case class Fetcher[Rdf<:RDF](store: Store[Rdf])(using ops: Ops[Rdf]) { - import ops.{given,*} - - -} +case class Fetcher[Rdf <: RDF](store: Store[Rdf])(using ops: Ops[Rdf]): + import ops.{given, *} @JSExportTopLevel("Fetcher") -object Fetcher { - -} +object Fetcher {} diff --git a/app/src/main/scala/run/cosy/solid/app/http/RDFMediaTypes.scala b/app/src/main/scala/run/cosy/solid/app/http/RDFMediaTypes.scala index 943baec..5c6539e 100644 --- a/app/src/main/scala/run/cosy/solid/app/http/RDFMediaTypes.scala +++ b/app/src/main/scala/run/cosy/solid/app/http/RDFMediaTypes.scala @@ -2,6 +2,4 @@ package run.cosy.solid.app.http // import org.http4s.headers.{Accept,*} -object RDFMediaTypes { - -} +object RDFMediaTypes {} diff --git a/app/src/main/scala/run/cosy/solid/app/http/Web.scala b/app/src/main/scala/run/cosy/solid/app/http/Web.scala index 284eda0..17166d4 100644 --- a/app/src/main/scala/run/cosy/solid/app/http/Web.scala +++ b/app/src/main/scala/run/cosy/solid/app/http/Web.scala @@ -1,5 +1,3 @@ package run.cosy.solid.app.http -class Web { - -} +class Web {} diff --git a/app/src/main/scala/solidapp/Example.scala b/app/src/main/scala/solidapp/Example.scala index 27943b4..5952cdf 100644 --- a/app/src/main/scala/solidapp/Example.scala +++ b/app/src/main/scala/solidapp/Example.scala @@ -15,68 +15,72 @@ import scala.scalajs.js.annotation.{JSExport, JSExportTopLevel} @JSExportTopLevel("Example") object Example: - import org.scalajs.dom - import dom.{document, html} - - val clnt: client.Client[IO] = FetchClientBuilder[IO].create - // val rdfHeaders = org.http4s.Headers - - import org.http4s.client.dsl.io.given - import org.http4s.headers.* - - def main(args: Array[String]): Unit = - document.addEventListener("DOMContentLoaded", { (e: dom.Event) => - println(e) - }) - - def setupUI(): Unit = - val button = document.createElement("button") - button.textContent = "Click me!" - button.addEventListener("click", { (e: dom.MouseEvent) => - addClickedMessage() - }) - document.body.appendChild(button) - appendPar(document.body, "Hello World") - end setupUI - - @JSExport - def addClickedMessage(): Unit = - val url: ParseResult[Uri] = urlEntry() - url.fold( - fail => appendPar(document.body, "could not parse url " + fail), - uri => { - appendPar(document.body, "You clicked to fetch " + uri) - onClick(uri) - } - ) - - def urlEntry(): ParseResult[Uri] = - val urlStr = input.value - println("URL=" + urlStr) - Uri.fromString(urlStr) - - def input: html.Input = document.getElementById("url").asInstanceOf[html.Input] - - def onClick(uri: Uri): Unit = - val utfStr: fs2.Stream[cats.effect.IO, String] = clnt.stream(req(uri)).flatMap(_.body) - .through(text.utf8.decode) - - val ios: fs2.Stream[cats.effect.IO, INothing] = utfStr.through(N3Parser.parse) - .chunks.foreach { chunk => - IO(appendPar(document.body, s"chunk size ${chunk.size} starts with ${chunk.head}")) - } - - ios.compile.lastOrError.unsafeRunAsync { - case Left(err) => appendPar(document.body, err.toString) - case Right(answer) => appendPar(document.body, "good answer") - } - end onClick - - def req(uri: Uri): Request[IO] = GET(uri, Accept(turtle.withQValue(QValue.One))) - - def appendPar(targetNode: dom.Node, text: String): Unit = - val parNode = document.createElement("p") - parNode.textContent = text - targetNode.appendChild(parNode) - -end Example \ No newline at end of file + import org.scalajs.dom + import dom.{document, html} + + val clnt: client.Client[IO] = FetchClientBuilder[IO].create + // val rdfHeaders = org.http4s.Headers + + import org.http4s.client.dsl.io.given + import org.http4s.headers.* + + def main(args: Array[String]): Unit = document.addEventListener( + "DOMContentLoaded", + { (e: dom.Event) => + println(e) + } + ) + + def setupUI(): Unit = + val button = document.createElement("button") + button.textContent = "Click me!" + button.addEventListener( + "click", + { (e: dom.MouseEvent) => + addClickedMessage() + } + ) + document.body.appendChild(button) + appendPar(document.body, "Hello World") + end setupUI + + @JSExport + def addClickedMessage(): Unit = + val url: ParseResult[Uri] = urlEntry() + url.fold( + fail => appendPar(document.body, "could not parse url " + fail), + uri => + appendPar(document.body, "You clicked to fetch " + uri) + onClick(uri) + ) + + def urlEntry(): ParseResult[Uri] = + val urlStr = input.value + println("URL=" + urlStr) + Uri.fromString(urlStr) + + def input: html.Input = document.getElementById("url").asInstanceOf[html.Input] + + def onClick(uri: Uri): Unit = + val utfStr: fs2.Stream[cats.effect.IO, String] = clnt.stream(req(uri)).flatMap(_.body) + .through(text.utf8.decode) + + val ios: fs2.Stream[cats.effect.IO, INothing] = utfStr.through(N3Parser.parse).chunks + .foreach { chunk => + IO(appendPar(document.body, s"chunk size ${chunk.size} starts with ${chunk.head}")) + } + + ios.compile.lastOrError.unsafeRunAsync { + case Left(err) => appendPar(document.body, err.toString) + case Right(answer) => appendPar(document.body, "good answer") + } + end onClick + + def req(uri: Uri): Request[IO] = GET(uri, Accept(turtle.withQValue(QValue.One))) + + def appendPar(targetNode: dom.Node, text: String): Unit = + val parNode = document.createElement("p") + parNode.textContent = text + targetNode.appendChild(parNode) + +end Example diff --git a/app/src/test/scala/run/cosy/solid/app/http/FetcherMunitTests.scala b/app/src/test/scala/run/cosy/solid/app/http/FetcherMunitTests.scala index 3ac68d1..2dcf603 100644 --- a/app/src/test/scala/run/cosy/solid/app/http/FetcherMunitTests.scala +++ b/app/src/test/scala/run/cosy/solid/app/http/FetcherMunitTests.scala @@ -9,12 +9,9 @@ import run.cosy.rdfjs.model.Quad import scala.scalajs.js +class FetcherMunitTests extends munit.FunSuite: + Fetcher.setupUI() -class FetcherMunitTests extends munit.FunSuite { - Fetcher.setupUI() - - test("HelloWorld") { - assert(document.querySelectorAll("p").count(_.textContent == "Hello World") == 1) - } - -} + test("HelloWorld") { + assert(document.querySelectorAll("p").count(_.textContent == "Hello World") == 1) + } diff --git a/build.sbt b/build.sbt index 48d614c..dda0860 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ import sbt.ThisBuild import sbtcrossproject.CrossPlugin.autoImport.{crossProject, CrossType} -import Dependencies._ +import Dependencies.* name := "SolidApp" ThisBuild / organization := "net.bblfish" @@ -214,7 +214,6 @@ lazy val scripts = crossProject(JVMPlatform).in(file("scripts")) crypto.bobcats.value classifier ("tests"), // bobcats test examples, crypto.bobcats.value classifier ("tests-sources"), // bobcats test examples soources, other.scalaUri.value, - http4s.ember_client.value, crypto.nimbusJWT_JDK.value, crypto.bouncyJCA_JDK.value diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/Cache.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/Cache.scala index 7cbc7c3..b5f84c5 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/Cache.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/Cache.scala @@ -1,143 +1,107 @@ package io.chrisdavenport.mules -import cats._ -import cats.syntax.all._ +import cats.* +import cats.syntax.all.* -trait Lookup[F[_], K, V]{ - def lookup(k: K): F[Option[V]] -} +trait Lookup[F[_], K, V]: + def lookup(k: K): F[Option[V]] -object Lookup { +object Lookup: - def mapValues[F[_]: Functor, K, A, B](l: Lookup[F, K, A])(f: A => B): Lookup[F, K, B] = - new Lookup[F, K, B]{ - def lookup(k: K): F[Option[B]] = l.lookup(k).map(_.map(f)) - } + def mapValues[F[_]: Functor, K, A, B](l: Lookup[F, K, A])(f: A => B): Lookup[F, K, B] = + new Lookup[F, K, B]: + def lookup(k: K): F[Option[B]] = l.lookup(k).map(_.map(f)) - def contramapKeys[F[_], K, B, A](l: Lookup[F, K, A])(f: B => K): Lookup[F, B, A] = - new Lookup[F, B, A]{ - def lookup(k: B): F[Option[A]] = l.lookup(f(k)) - } + def contramapKeys[F[_], K, B, A](l: Lookup[F, K, A])(f: B => K): Lookup[F, B, A] = + new Lookup[F, B, A]: + def lookup(k: B): F[Option[A]] = l.lookup(f(k)) - def evalMap[F[_]: Monad, K, A, B](l: Lookup[F, K, A])(f: A => F[B]): Lookup[F, K, B] = - new Lookup[F, K, B]{ - def lookup(k: K): F[Option[B]] = l.lookup(k).flatMap(_.traverse(f)) - } + def evalMap[F[_]: Monad, K, A, B](l: Lookup[F, K, A])(f: A => F[B]): Lookup[F, K, B] = + new Lookup[F, K, B]: + def lookup(k: K): F[Option[B]] = l.lookup(k).flatMap(_.traverse(f)) - def mapK[F[_], G[_], K, V](l: Lookup[F, K, V])(fk: F ~> G): Lookup[G, K, V] = - new Lookup[G, K, V]{ - def lookup(k: K): G[Option[V]] = fk(l.lookup(k)) - } -} + def mapK[F[_], G[_], K, V](l: Lookup[F, K, V])(fk: F ~> G): Lookup[G, K, V] = + new Lookup[G, K, V]: + def lookup(k: K): G[Option[V]] = fk(l.lookup(k)) -trait Get[F[_], K, V]{ - def get(k: K): F[V] -} +trait Get[F[_], K, V]: + def get(k: K): F[V] -object Get { +object Get: - def mapValues[F[_]: Functor, K, A, B](l: Get[F, K, A])(f: A => B): Get[F, K, B] = - new Get[F, K, B]{ - def get(k: K): F[B] = l.get(k).map(f) - } + def mapValues[F[_]: Functor, K, A, B](l: Get[F, K, A])(f: A => B): Get[F, K, B] = + new Get[F, K, B]: + def get(k: K): F[B] = l.get(k).map(f) - def contramapKeys[F[_], K, B, A](l: Get[F, K, A])(g: B => K): Get[F, B, A] = - new Get[F, B, A]{ + def contramapKeys[F[_], K, B, A](l: Get[F, K, A])(g: B => K): Get[F, B, A] = new Get[F, B, A]: def get(k: B): F[A] = l.get(g(k)) - } - def evalMap[F[_]: Monad, K, A, B](l: Get[F, K, A])(f: A => F[B]): Get[F, K, B] = - new Get[F, K, B]{ - def get(k: K): F[B] = l.get(k).flatMap(f) - } + def evalMap[F[_]: Monad, K, A, B](l: Get[F, K, A])(f: A => F[B]): Get[F, K, B] = + new Get[F, K, B]: + def get(k: K): F[B] = l.get(k).flatMap(f) - def mapK[F[_], G[_], K, V](g: Get[F, K, V])(fk: F ~> G): Get[G, K, V] = - new Get[G, K, V]{ + def mapK[F[_], G[_], K, V](g: Get[F, K, V])(fk: F ~> G): Get[G, K, V] = new Get[G, K, V]: def get(k: K): G[V] = fk(g.get(k)) - } -} +trait Insert[F[_], K, V]: + def insert(k: K, v: V): F[Unit] -trait Insert[F[_], K, V]{ - def insert(k: K, v: V): F[Unit] -} +object Insert: -object Insert { + def contramapValues[F[_], K, A, B](i: Insert[F, K, A])(g: B => A): Insert[F, K, B] = + new Insert[F, K, B]: + def insert(k: K, v: B): F[Unit] = i.insert(k, g(v)) - def contramapValues[F[_], K, A, B](i: Insert[F, K, A])(g: B => A): Insert[F, K, B] = - new Insert[F, K, B]{ - def insert(k: K, v: B): F[Unit] = i.insert(k, g(v)) - } + def contramapKeys[F[_], A, B, V](i: Insert[F, A, V])(g: B => A): Insert[F, B, V] = + new Insert[F, B, V]: + def insert(k: B, v: V): F[Unit] = i.insert(g(k), v) - def contramapKeys[F[_], A, B, V](i: Insert[F, A, V])(g: B => A): Insert[F, B, V] = - new Insert[F, B, V]{ - def insert(k: B, v: V): F[Unit] = i.insert(g(k), v) - } + def mapK[F[_], G[_], K, V](i: Insert[F, K, V])(fk: F ~> G): Insert[G, K, V] = + new Insert[G, K, V]: + def insert(k: K, v: V): G[Unit] = fk(i.insert(k, v)) - def mapK[F[_], G[_], K, V](i: Insert[F, K, V])(fk: F ~> G): Insert[G, K, V] = - new Insert[G, K, V]{ - def insert(k: K, v: V): G[Unit] = fk(i.insert(k, v)) - } +trait Delete[F[_], K]: + def delete(k: K): F[Unit] -} - -trait Delete[F[_], K]{ - def delete(k: K): F[Unit] -} - -object Delete { - def contramap[F[_], A, B](d: Delete[F, A])(g: B => A): Delete[F, B] = - new Delete[F, B]{ +object Delete: + def contramap[F[_], A, B](d: Delete[F, A])(g: B => A): Delete[F, B] = new Delete[F, B]: def delete(k: B) = d.delete(g(k)) - } - def mapK[F[_], G[_], K](d: Delete[F, K])(fk: F ~> G): Delete[G, K] = - new Delete[G, K]{ + def mapK[F[_], G[_], K](d: Delete[F, K])(fk: F ~> G): Delete[G, K] = new Delete[G, K]: def delete(k: K): G[Unit] = fk(d.delete(k)) - } -} - /** Local Search function into the cache */ trait LocalSearch[F[_], K, V]: - /* find closest value from key going down the URL hierarchy that satisifies predicate. - * the domain is an Option[K], so the function can decided what to do if a node does not exist - */ - def findClosest(k: K)(predicate: Option[K] => Boolean): F[Option[V]] - -trait Cache[F[_], K, V] - extends Lookup[F, K, V] - with Insert[F, K, V] - with Delete[F, K] - -object Cache { - def imapValues[F[_]: Functor, K, A, B](cache: Cache[F, K, A])(f: A => B, g: B => A): Cache[F, K, B] = - new Cache[F, K, B]{ + /* find closest value from key going down the URL hierarchy that satisifies predicate. + * the domain is an Option[K], so the function can decided what to do if a node does not exist + */ + def findClosest(k: K)(predicate: Option[K] => Boolean): F[Option[V]] + +trait Cache[F[_], K, V] extends Lookup[F, K, V] with Insert[F, K, V] with Delete[F, K] + +object Cache: + def imapValues[F[_]: Functor, K, A, B]( + cache: Cache[F, K, A] + )(f: A => B, g: B => A): Cache[F, K, B] = new Cache[F, K, B]: def lookup(k: K): F[Option[B]] = cache.lookup(k).map(_.map(f)) def insert(k: K, v: B): F[Unit] = cache.insert(k, g(v)) def delete(k: K): F[Unit] = cache.delete(k) - } - - def contramapKeys[F[_], K1, K2, V](c: Cache[F, K1, V])(g: K2 => K1): Cache[F, K2, V] = - new Cache[F, K2, V] { - def lookup(k: K2): F[Option[V]] = c.lookup(g(k)) - def insert(k: K2, v: V): F[Unit] = c.insert(g(k), v) - def delete(k: K2): F[Unit] = c.delete(g(k)) - } - - def evalIMap[F[_]: Monad, K, V1, V2]( - c: Cache[F, K, V1] - )(f: V1 => F[V2], g: V2 => V1): Cache[F, K, V2] = - new Cache[F, K, V2] { + + def contramapKeys[F[_], K1, K2, V](c: Cache[F, K1, V])(g: K2 => K1): Cache[F, K2, V] = + new Cache[F, K2, V]: + def lookup(k: K2): F[Option[V]] = c.lookup(g(k)) + def insert(k: K2, v: V): F[Unit] = c.insert(g(k), v) + def delete(k: K2): F[Unit] = c.delete(g(k)) + + def evalIMap[F[_]: Monad, K, V1, V2]( + c: Cache[F, K, V1] + )(f: V1 => F[V2], g: V2 => V1): Cache[F, K, V2] = new Cache[F, K, V2]: def lookup(k: K): F[Option[V2]] = c.lookup(k).flatMap(_.traverse(f)) def insert(k: K, v: V2): F[Unit] = c.insert(k, g(v)) def delete(k: K): F[Unit] = c.delete(k) - } - - def mapK[F[_], G[_], K, V](cache: Cache[F, K, V])(fk: F ~> G): Cache[G, K, V] = - new Cache[G, K, V]{ - def lookup(k: K): G[Option[V]] = fk(cache.lookup(k)) - def insert(k: K, v: V): G[Unit] = fk(cache.insert(k, v)) - def delete(k: K): G[Unit] = fk(cache.delete(k)) - } -} \ No newline at end of file + + def mapK[F[_], G[_], K, V](cache: Cache[F, K, V])(fk: F ~> G): Cache[G, K, V] = + new Cache[G, K, V]: + def lookup(k: K): G[Option[V]] = fk(cache.lookup(k)) + def insert(k: K, v: V): G[Unit] = fk(cache.insert(k, v)) + def delete(k: K): G[Unit] = fk(cache.delete(k)) diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala index ca0e59d..6e48f8b 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala @@ -40,7 +40,8 @@ object CacheItem: requestMethod: Method, response: CachedResponse[T], expires: Option[HttpDate] - ): F[CacheItem[T]] = HttpDate.current[F].map(date => new CacheItem(requestMethod, date, expires, response)) + ): F[CacheItem[T]] = HttpDate.current[F] + .map(date => new CacheItem(requestMethod, date, expires, response)) private[http4s] final case class Age(val deltaSeconds: Long) extends AnyVal private[http4s] object Age: diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala index 67f69fd..17762f8 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CachedResponse.scala @@ -33,7 +33,7 @@ final case class CachedResponse[T]( body: Option[T] ): def withHeaders(headers: Headers): CachedResponse[T] = this.copy(headers = headers) - + def map[S](f: T => S): CachedResponse[S] = this.copy(body = body.map(f)) object CachedResponse: diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala index 2f8a3e9..58855c9 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala @@ -50,10 +50,8 @@ case class Caching[F[_]: Concurrent: Clock, T]( then fk(CachedResponse.errorResponse[T](Status.GatewayTimeout).pure[F]) else app.run(req).flatMap(resp => fk(withResponse(req, resp))) case Some(item) => CacheRules.cachedObjectOk(req, item, now) match - case Some(item) => - fk(item.response.pure[F]) - case None => - app.run( + case Some(item) => fk(item.response.pure[F]) + case None => app.run( req.putHeaders( CacheRules.getIfMatch(item.response).map(modelledHeadersToRaw(_)).toSeq* ).putHeaders( diff --git a/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala b/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala index b5279e0..7a5e0a4 100644 --- a/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala +++ b/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala @@ -87,9 +87,9 @@ object DirTree: end unzipAlong /** find the closest node matching `select` going backwards from the closest node we have - * leading to path. So if we want - * but we have and but only that content - * at the latter resource matches, then we will get that. + * leading to path. So if we want but we have + * and but only that content at the latter + * resource matches, then we will get that. */ def findClosest(path: Path)(select: X => Boolean): Option[X] = unzipAlong(path) match case (Right(dt), zpath) => dt.head +: zpath.map(_.from.head) find select diff --git a/cache/shared/src/main/scala/run/cosy/http/cache/TreeDirCache.scala b/cache/shared/src/main/scala/run/cosy/http/cache/TreeDirCache.scala index e7aacd8..d37725e 100644 --- a/cache/shared/src/main/scala/run/cosy/http/cache/TreeDirCache.scala +++ b/cache/shared/src/main/scala/run/cosy/http/cache/TreeDirCache.scala @@ -19,7 +19,7 @@ package run.cosy.http.cache import cats.effect.kernel.{Ref, Sync} import cats.syntax.all.* import cats.{FlatMap, MonadError} -import io.chrisdavenport.mules.{Cache,LocalSearch} +import io.chrisdavenport.mules.{Cache, LocalSearch} import org.http4s.Uri import run.cosy.http.cache.DirTree.* import run.cosy.http.cache.TreeDirCache.WebCache @@ -107,4 +107,3 @@ case class TreeDirCache[F[_], X]( // server = (scheme, auth) // tree <- F.fromOption(webCache.get(server), ServerNotFound(k)) // yield tree.findClosest(k.path.segments)(predicate).flatten - diff --git a/cache/shared/src/test/scala/run/cosy/http/cache/WebTest.scala b/cache/shared/src/test/scala/run/cosy/http/cache/WebTest.scala index 24d48f9..d27c256 100644 --- a/cache/shared/src/test/scala/run/cosy/http/cache/WebTest.scala +++ b/cache/shared/src/test/scala/run/cosy/http/cache/WebTest.scala @@ -122,11 +122,11 @@ object WebTest: |""".stripMargin val bblBlogRootContainer = """ - |@prefix ldp: . - | - |<> a ldp:BasicContainer; - | ldp:contains <2023/> . - |""".stripMargin + |@prefix ldp: . + | + |<> a ldp:BasicContainer; + | ldp:contains <2023/> . + |""".stripMargin val bblWorldAtPeace = "Hello World!" val bblBlogVirgin = "Play in three acts" @@ -183,7 +183,8 @@ object WebTest: bblBlogVirgin, headers("birth", Some("/people/henry/blog/"), MediaType.text.plain) ) - case HEAD -> Root / "people" / "henry" / "blog" / "" => Async[F].pure(bblBlogDir(req).copy(entity = Entity.empty)) + case HEAD -> Root / "people" / "henry" / "blog" / "" => Async[F] + .pure(bblBlogDir(req).copy(entity = Entity.empty)) case GET -> Root / "people" / "henry" / "blog" / "" => Async[F].pure(bblBlogDir(req)) case GET -> Root / "people" / "henry" / "blog" / "2023" / "04" / "01" / "world-at-peace" => OK[F]( diff --git a/free/shared/net/bblfish/ldp/cmd/LDPCmd.scala b/free/shared/net/bblfish/ldp/cmd/LDPCmd.scala index b6082bc..46b61f9 100644 --- a/free/shared/net/bblfish/ldp/cmd/LDPCmd.scala +++ b/free/shared/net/bblfish/ldp/cmd/LDPCmd.scala @@ -6,27 +6,19 @@ import org.http4s.Headers import org.http4s.Response import cats.free.Free -/** - * LDP Commands for a free monad and free applicatives. - * (see "An Intuitive Guide to Combining Free Monads and Free Applicatives" - * https://twitter.com/bblfish/status/1587052879986180097 ) - * - * What do we need? - * We need GET/POST/PUT/DELETE/QUERY - * And we are *very* often interested in RDF Graphs being returned. - * But we canno - * Similar to an Http Request but where we sometimes want to be able to avoid - * the serialisation part. +/** LDP Commands for a free monad and free applicatives. (see "An Intuitive Guide to Combining Free + * Monads and Free Applicatives" https://twitter.com/bblfish/status/1587052879986180097 ) * + * What do we need? We need GET/POST/PUT/DELETE/QUERY And we are *very* often interested in RDF + * Graphs being returned. But we canno Similar to an Http Request but where we sometimes want to be + * able to avoid the serialisation part. */ sealed trait LdpCmd[A]: - /* All comands are aimed at a URL. - * todo: consider relative URLs */ - def url: Uri - + /* All comands are aimed at a URL. + * todo: consider relative URLs */ + def url: Uri -/** - * Get request from URL where we expect a graph response. +/** Get request from URL where we expect a graph response. * * We don't start with a generalised version, to keep things simple. * @@ -40,33 +32,30 @@ sealed trait LdpCmd[A]: case class GetGraph[A](url: Uri, k: GraphResponse => A) extends LdpCmd[A] /** A response to a request. The Metadata tells if the response succeeded, what the problems may - * have been, etc... The content, is an attempted parse of the stream to the object type. - * (todo?: generalise the type of the content to quite a lot more types, such as DataSets, - * Images, streams, ...) + * have been, etc... The content, is an attempted parse of the stream to the object type. (todo?: + * generalise the type of the content to quite a lot more types, such as DataSets, Images, streams, + * ...) * - * todo: should the result type be generic? In which case this would be a - * generalisation of http4s. + * todo: should the result type be generic? In which case this would be a generalisation of http4s. */ -case class GraphResponse[R<:RDF](meta: Meta, content: Try[RDF.Graph[R]]) - -/** - * Metadata on the resource from the server. - * @param url the URL of the resource - * @param status status of the response - * @param headers Headers on the response - * +case class GraphResponse[R <: RDF](meta: Meta, content: Try[RDF.Graph[R]]) + +/** Metadata on the resource from the server. + * @param url + * the URL of the resource + * @param status + * status of the response + * @param headers + * Headers on the response */ case class Meta(url: Uri, status: Status = Status.Ok, headers: Headers) - - object LdpCmd: // type NamedGraph = (Uri, Rdf#Graph) // type NamedGraphs = Map[Uri,Rdf#Graph] - type Script[A] = cats.free.Free[LDPCmd, A] + type Script[A] = cats.free.Free[LDPCmd, A] - def pure[A](a: A): Script[A] = Free.pure(a) + def pure[A](a: A): Script[A] = Free.pure(a) - def get(key: Uri): Script[Response] = - Free.liftF[LdpCmd, Response](Get[Response](key, identity)) + def get(key: Uri): Script[Response] = Free.liftF[LdpCmd, Response](Get[Response](key, identity)) diff --git a/ioExt4s/shared/src/main/scala/run/cosy/ld/http4s/RDFDecoders.scala b/ioExt4s/shared/src/main/scala/run/cosy/ld/http4s/RDFDecoders.scala index 9792e7c..794b974 100644 --- a/ioExt4s/shared/src/main/scala/run/cosy/ld/http4s/RDFDecoders.scala +++ b/ioExt4s/shared/src/main/scala/run/cosy/ld/http4s/RDFDecoders.scala @@ -65,7 +65,7 @@ class RDFDecoders[F[_], Rdf <: RDF](using } ) } - + import MediaType.{application, text} import MediaType.application.`ld+json` diff --git a/ioExt4s/shared/src/test/scala/run/cosy/web/util/UrlUtilTest.scala b/ioExt4s/shared/src/test/scala/run/cosy/web/util/UrlUtilTest.scala index ade21b6..897a829 100644 --- a/ioExt4s/shared/src/test/scala/run/cosy/web/util/UrlUtilTest.scala +++ b/ioExt4s/shared/src/test/scala/run/cosy/web/util/UrlUtilTest.scala @@ -3,7 +3,7 @@ package run.cosy.web.util import io.lemonlabs.uri as ll import run.cosy.web.util.UrlUtil.* -class UrlUtilTest extends munit.FunSuite { +class UrlUtilTest extends munit.FunSuite: def p(s: String) = ll.AbsoluteUrl.parseOption(s) val bblRoot = p("https://bblfish.net/") val bblCard = p("https://bblfish.net/people/henry/card#me") @@ -12,12 +12,10 @@ class UrlUtilTest extends munit.FunSuite { val bbldoc = p("https://bblfish.net") test("parent test") { - assert(henry.isDefined) - assertEquals(bblCard.flatMap(_.parent), henry) - assert(henry.isDefined) - assertEquals(henry.flatMap(_.parent), ppl) - assertEquals(ppl.flatMap(_.parent), bblRoot) - assertEquals(bblRoot.flatMap(_.parent), None) + assert(henry.isDefined) + assertEquals(bblCard.flatMap(_.parent), henry) + assert(henry.isDefined) + assertEquals(henry.flatMap(_.parent), ppl) + assertEquals(ppl.flatMap(_.parent), bblRoot) + assertEquals(bblRoot.flatMap(_.parent), None) } - -} diff --git a/n3js/src/main/scala/n3js/n3/mod/BaseFormat.scala b/n3js/src/main/scala/n3js/n3/mod/BaseFormat.scala index 05e4dd7..0ec3801 100644 --- a/n3js/src/main/scala/n3js/n3/mod/BaseFormat.scala +++ b/n3js/src/main/scala/n3js/n3/mod/BaseFormat.scala @@ -5,32 +5,33 @@ import scala.scalajs.js import scala.scalajs.js.`|` import scala.scalajs.js.annotation.{JSGlobalScope, JSGlobal, JSImport, JSName, JSBracketAccess} -/* Rewritten from type alias, can be one of: +/* Rewritten from type alias, can be one of: - n3js.n3.n3Strings.Turtle - n3js.n3.n3Strings.TriG - n3js.n3.n3Strings.`N-Triples` - n3js.n3.n3Strings.`N-Quads` - n3js.n3.n3Strings.N3 - n3js.n3.n3Strings.Notation3 -*/ + */ trait BaseFormat extends StObject -object BaseFormat { - - @scala.inline - def `N-Quads`: n3js.n3.n3Strings.`N-Quads` = "N-Quads".asInstanceOf[n3js.n3.n3Strings.`N-Quads`] - - @scala.inline - def `N-Triples`: n3js.n3.n3Strings.`N-Triples` = "N-Triples".asInstanceOf[n3js.n3.n3Strings.`N-Triples`] - - @scala.inline - def N3: n3js.n3.n3Strings.N3 = "N3".asInstanceOf[n3js.n3.n3Strings.N3] - - @scala.inline - def Notation3: n3js.n3.n3Strings.Notation3 = "Notation3".asInstanceOf[n3js.n3.n3Strings.Notation3] - - @scala.inline - def TriG: n3js.n3.n3Strings.TriG = "TriG".asInstanceOf[n3js.n3.n3Strings.TriG] - - @scala.inline - def Turtle: n3js.n3.n3Strings.Turtle = "Turtle".asInstanceOf[n3js.n3.n3Strings.Turtle] -} +object BaseFormat: + + @scala.inline + def `N-Quads`: n3js.n3.n3Strings.`N-Quads` = "N-Quads".asInstanceOf[n3js.n3.n3Strings.`N-Quads`] + + @scala.inline + def `N-Triples`: n3js.n3.n3Strings.`N-Triples` = "N-Triples" + .asInstanceOf[n3js.n3.n3Strings.`N-Triples`] + + @scala.inline + def N3: n3js.n3.n3Strings.N3 = "N3".asInstanceOf[n3js.n3.n3Strings.N3] + + @scala.inline + def Notation3: n3js.n3.n3Strings.Notation3 = "Notation3" + .asInstanceOf[n3js.n3.n3Strings.Notation3] + + @scala.inline + def TriG: n3js.n3.n3Strings.TriG = "TriG".asInstanceOf[n3js.n3.n3Strings.TriG] + + @scala.inline + def Turtle: n3js.n3.n3Strings.Turtle = "Turtle".asInstanceOf[n3js.n3.n3Strings.Turtle] diff --git a/n3js/src/main/scala/n3js/n3/mod/Lexer.scala b/n3js/src/main/scala/n3js/n3/mod/Lexer.scala index 5e67fbf..93e3470 100644 --- a/n3js/src/main/scala/n3js/n3/mod/Lexer.scala +++ b/n3js/src/main/scala/n3js/n3/mod/Lexer.scala @@ -7,62 +7,61 @@ import scala.scalajs.js.annotation.JSImport import scala.scalajs.js.ThisFunction1 import scala.scalajs.js.annotation.JSName -/** + This is generated from the typescript module for N3 JS library, but we - * added a few methods and intend to change the behavior. Lexer objects should - * therefore only be created with apply method in the companion object. +/** + This is generated from the typescript module for N3 JS library, but we added a few methods and + * intend to change the behavior. Lexer objects should therefore only be created with apply method + * in the companion object. * - * Because we need to rewrite some functions, we add the protected functions - * starting with an `_`. + * Because we need to rewrite some functions, we add the protected functions starting with an `_`. */ @JSImport("n3", "Lexer") @js.native class Lexer() extends StObject: - def this(options: LexerOptions) = this() + def this(options: LexerOptions) = this() - def tokenize(input: String): js.Array[Token] = js.native - def tokenize(input: String, callback: TokenCallback): Unit = js.native + def tokenize(input: String): js.Array[Token] = js.native + def tokenize(input: String, callback: TokenCallback): Unit = js.native // def tokenize(input: EventEmitter, callback: TokenCallback): Unit = js.native - // - // protected functions and vars we need access to in our code - // - var _line: Int = js.native - var _input: js.UndefOr[String] = js.native - var _callback: js.UndefOr[TokenCallback] = js.native - def _tokenizeToEnd(callback: TokenCallback, inputFinished: Boolean): Unit = js.native + // + // protected functions and vars we need access to in our code + // + var _line: Int = js.native + var _input: js.UndefOr[String] = js.native + var _callback: js.UndefOr[TokenCallback] = js.native + def _tokenizeToEnd(callback: TokenCallback, inputFinished: Boolean): Unit = js.native - // - // functions we create - // - var setCallback: js.ThisFunction1[Lexer,TokenCallback, Unit] = js.native + // + // functions we create + // + var setCallback: js.ThisFunction1[Lexer, TokenCallback, Unit] = js.native - //this function is added during object creation - def tokenizeChunk(input: String, inputFinished: Boolean): Unit = js.native + // this function is added during object creation + def tokenizeChunk(input: String, inputFinished: Boolean): Unit = js.native end Lexer object Lexer: - def apply(options: LexerOptions = LexerOptions()): Lexer = - val lex: Lexer = new Lexer(options) - lex._input = "" - //could one set these on the prototype? - lex.asInstanceOf[js.Dynamic].updateDynamic("setCallback")(setCallback) - lex.asInstanceOf[js.Dynamic].updateDynamic("tokenizeChunk")(tokenizeChunk) - lex + def apply(options: LexerOptions = LexerOptions()): Lexer = + val lex: Lexer = new Lexer(options) + lex._input = "" + // could one set these on the prototype? + lex.asInstanceOf[js.Dynamic].updateDynamic("setCallback")(setCallback) + lex.asInstanceOf[js.Dynamic].updateDynamic("tokenizeChunk")(tokenizeChunk) + lex - //must be called on initialisation - val setCallback: js.ThisFunction1[Lexer, TokenCallback, Unit] = - (thiz: Lexer, tokCbk: TokenCallback) => - thiz._line = 1 - thiz._input = "" - thiz._callback = tokCbk - () + // must be called on initialisation + val setCallback: js.ThisFunction1[Lexer, TokenCallback, Unit] = + (thiz: Lexer, tokCbk: TokenCallback) => + thiz._line = 1 + thiz._input = "" + thiz._callback = tokCbk + () - //this is the function f in `input.on('data', f)` - val tokenizeChunk: js.ThisFunction2[Lexer, String, Boolean, Unit] = - (thiz: Lexer, chunk: String, end: Boolean) => - thiz._input = thiz._input.getOrElse("") + chunk - thiz._callback.map(tcb => thiz._tokenizeToEnd(tcb, end)) - () + // this is the function f in `input.on('data', f)` + val tokenizeChunk: js.ThisFunction2[Lexer, String, Boolean, Unit] = + (thiz: Lexer, chunk: String, end: Boolean) => + thiz._input = thiz._input.getOrElse("") + chunk + thiz._callback.map(tcb => thiz._tokenizeToEnd(tcb, end)) + () -end Lexer \ No newline at end of file +end Lexer diff --git a/n3js/src/main/scala/n3js/n3/mod/LexerOptions.scala b/n3js/src/main/scala/n3js/n3/mod/LexerOptions.scala index 4d472fb..9996fc2 100644 --- a/n3js/src/main/scala/n3js/n3/mod/LexerOptions.scala +++ b/n3js/src/main/scala/n3js/n3/mod/LexerOptions.scala @@ -6,41 +6,39 @@ import scala.scalajs.js.`|` import scala.scalajs.js.annotation.{JSGlobalScope, JSGlobal, JSImport, JSName, JSBracketAccess} @js.native -trait LexerOptions extends StObject { - - var comments: js.UndefOr[Boolean] = js.native - - var lineMode: js.UndefOr[Boolean] = js.native - - var n3: js.UndefOr[Boolean] = js.native -} -object LexerOptions { - - @scala.inline - def apply(): LexerOptions = { - val __obj = js.Dynamic.literal() - __obj.asInstanceOf[LexerOptions] - } - - @scala.inline - implicit class LexerOptionsMutableBuilder[Self <: LexerOptions] (val x: Self) extends AnyVal { - - @scala.inline - def setComments(value: Boolean): Self = StObject.set(x, "comments", value.asInstanceOf[js.Any]) - - @scala.inline - def setCommentsUndefined: Self = StObject.set(x, "comments", js.undefined) - - @scala.inline - def setLineMode(value: Boolean): Self = StObject.set(x, "lineMode", value.asInstanceOf[js.Any]) - - @scala.inline - def setLineModeUndefined: Self = StObject.set(x, "lineMode", js.undefined) - - @scala.inline - def setN3(value: Boolean): Self = StObject.set(x, "n3", value.asInstanceOf[js.Any]) - - @scala.inline - def setN3Undefined: Self = StObject.set(x, "n3", js.undefined) - } -} +trait LexerOptions extends StObject: + + var comments: js.UndefOr[Boolean] = js.native + + var lineMode: js.UndefOr[Boolean] = js.native + + var n3: js.UndefOr[Boolean] = js.native +object LexerOptions: + + @scala.inline + def apply(): LexerOptions = + val __obj = js.Dynamic.literal() + __obj.asInstanceOf[LexerOptions] + + @scala.inline + implicit class LexerOptionsMutableBuilder[Self <: LexerOptions](val x: Self) extends AnyVal: + + @scala.inline + def setComments(value: Boolean): Self = StObject + .set(x, "comments", value.asInstanceOf[js.Any]) + + @scala.inline + def setCommentsUndefined: Self = StObject.set(x, "comments", js.undefined) + + @scala.inline + def setLineMode(value: Boolean): Self = StObject + .set(x, "lineMode", value.asInstanceOf[js.Any]) + + @scala.inline + def setLineModeUndefined: Self = StObject.set(x, "lineMode", js.undefined) + + @scala.inline + def setN3(value: Boolean): Self = StObject.set(x, "n3", value.asInstanceOf[js.Any]) + + @scala.inline + def setN3Undefined: Self = StObject.set(x, "n3", js.undefined) diff --git a/n3js/src/main/scala/n3js/n3/mod/Logger.scala b/n3js/src/main/scala/n3js/n3/mod/Logger.scala index dd4324c..3f1490a 100644 --- a/n3js/src/main/scala/n3js/n3/mod/Logger.scala +++ b/n3js/src/main/scala/n3js/n3/mod/Logger.scala @@ -6,8 +6,7 @@ import scala.scalajs.js.`|` import scala.scalajs.js.annotation.{JSGlobalScope, JSGlobal, JSImport, JSName, JSBracketAccess} @js.native -trait Logger extends StObject { - - def apply(message: js.Any, optionalParams: js.Any*): Unit = js.native - def apply(message: Unit, optionalParams: js.Any*): Unit = js.native -} +trait Logger extends StObject: + + def apply(message: js.Any, optionalParams: js.Any*): Unit = js.native + def apply(message: Unit, optionalParams: js.Any*): Unit = js.native diff --git a/n3js/src/main/scala/n3js/n3/mod/MimeType.scala b/n3js/src/main/scala/n3js/n3/mod/MimeType.scala index 0eee31b..da23410 100644 --- a/n3js/src/main/scala/n3js/n3/mod/MimeType.scala +++ b/n3js/src/main/scala/n3js/n3/mod/MimeType.scala @@ -5,28 +5,29 @@ import scala.scalajs.js import scala.scalajs.js.`|` import scala.scalajs.js.annotation.{JSGlobalScope, JSGlobal, JSImport, JSName, JSBracketAccess} -/* Rewritten from type alias, can be one of: +/* Rewritten from type alias, can be one of: - n3js.n3.n3Strings.application - n3js.n3.n3Strings.example - n3js.n3.n3Strings.text - n3js.n3.n3Strings.message - n3js.n3.n3Strings.multipart -*/ + */ trait MimeType extends StObject -object MimeType { - - @scala.inline - def application: n3js.n3.n3Strings.application = "application".asInstanceOf[n3js.n3.n3Strings.application] - - @scala.inline - def example: n3js.n3.n3Strings.example = "example".asInstanceOf[n3js.n3.n3Strings.example] - - @scala.inline - def message: n3js.n3.n3Strings.message = "message".asInstanceOf[n3js.n3.n3Strings.message] - - @scala.inline - def multipart: n3js.n3.n3Strings.multipart = "multipart".asInstanceOf[n3js.n3.n3Strings.multipart] - - @scala.inline - def text: n3js.n3.n3Strings.text = "text".asInstanceOf[n3js.n3.n3Strings.text] -} +object MimeType: + + @scala.inline + def application: n3js.n3.n3Strings.application = "application" + .asInstanceOf[n3js.n3.n3Strings.application] + + @scala.inline + def example: n3js.n3.n3Strings.example = "example".asInstanceOf[n3js.n3.n3Strings.example] + + @scala.inline + def message: n3js.n3.n3Strings.message = "message".asInstanceOf[n3js.n3.n3Strings.message] + + @scala.inline + def multipart: n3js.n3.n3Strings.multipart = "multipart" + .asInstanceOf[n3js.n3.n3Strings.multipart] + + @scala.inline + def text: n3js.n3.n3Strings.text = "text".asInstanceOf[n3js.n3.n3Strings.text] diff --git a/n3js/src/main/scala/n3js/n3/mod/Parser.scala b/n3js/src/main/scala/n3js/n3/mod/Parser.scala index 1504050..289f8b2 100644 --- a/n3js/src/main/scala/n3js/n3/mod/Parser.scala +++ b/n3js/src/main/scala/n3js/n3/mod/Parser.scala @@ -3,91 +3,101 @@ package n3js.n3.mod import org.scalablytyped.runtime.StObject import scala.scalajs.js import scala.scalajs.js.annotation.JSImport -import run.cosy.rdfjs.model.{Quad,NamedNode} +import run.cosy.rdfjs.model.{Quad, NamedNode} import n3js.n3.mod.ParserOptions @JSImport("n3", "Parser") @js.native class Parser() extends StObject: - def this(options: ParserOptions) = this() - import Parser.TokenFn + def this(options: ParserOptions) = this() + import Parser.TokenFn - def parse(input: String): js.Array[Quad] = js.native + def parse(input: String): js.Array[Quad] = js.native // def parse(input: String, callback: ParseCallback[Q]): Unit = js.native - def parse( - input: String, - callback: ParseCallback, - prefixCallback: PrefixCallback - ): Unit = js.native + def parse( + input: String, + callback: ParseCallback, + prefixCallback: PrefixCallback + ): Unit = js.native // def parse(input: String, callback: Null, prefixCallback: PrefixCallback): js.Array[Q] = js.native // def parse(input: String, callback: Unit, prefixCallback: PrefixCallback): js.Array[Q] = js.native + // + // existing JS fields we make explicit + // + val _readInTopContext: TokenFn = js.native + var _readCallback: TokenFn = js.native + var _sparqlStyle: Boolean = js.native + var _callback: ParseCallback = js.native + var _prefixes: js.Object = js.native + var _prefixCallback: PrefixCallback = js.native + var _quantified: js.Object = js.native + var _inversePredicate: Boolean = js.native + var _lexer: Lexer = js.native + // for testing + var _emit: js.ThisFunction4[Parser, Quad.Subject, Quad.Predicate, Quad.Object, js.UndefOr[ + Quad.Graph + ], Unit] = js.native + val _quad + : js.Function4[Quad.Subject, Quad.Predicate, Quad.Object, js.UndefOr[Quad.Graph], Quad] = + js.native - // - // existing JS fields we make explicit - // - val _readInTopContext: TokenFn = js.native - var _readCallback: TokenFn = js.native - var _sparqlStyle: Boolean = js.native - var _callback: ParseCallback = js.native - var _prefixes: js.Object = js.native - var _prefixCallback: PrefixCallback = js.native - var _quantified: js.Object = js.native - var _inversePredicate: Boolean = js.native - var _lexer: Lexer = js.native - //for testing - var _emit: js.ThisFunction4[Parser, Quad.Subject, Quad.Predicate, Quad.Object, js.UndefOr[Quad.Graph], Unit] = js.native - val _quad: js.Function4[Quad.Subject, Quad.Predicate, Quad.Object, js.UndefOr[Quad.Graph], Quad] = js.native + // + // functions we add + // - // - // functions we add - // + var setCallbacks: js.ThisFunction2[Parser, ParseCallback, PrefixCallback, Unit] = js.native - var setCallbacks : js.ThisFunction2[Parser, ParseCallback, PrefixCallback, Unit] = js.native - - // send the next string chunk to this parser. Need to create this method on construction. - def parseChunk(chunk: String, end: Boolean): Unit = js.native + // send the next string chunk to this parser. Need to create this method on construction. + def parseChunk(chunk: String, end: Boolean): Unit = js.native end Parser object Parser: - type TokenFn = js.ThisFunction1[Parser,Token,Any] + type TokenFn = js.ThisFunction1[Parser, Token, Any] - def apply(options: ParserOptions): Parser = - val p = new Parser(options) - p.asInstanceOf[js.Dynamic].updateDynamic("setCallbacks")(setCallbacks) - p.asInstanceOf[js.Dynamic].updateDynamic("parseChunk")(parseChunk) - p._lexer = Lexer() - p + def apply(options: ParserOptions): Parser = + val p = new Parser(options) + p.asInstanceOf[js.Dynamic].updateDynamic("setCallbacks")(setCallbacks) + p.asInstanceOf[js.Dynamic].updateDynamic("parseChunk")(parseChunk) + p._lexer = Lexer() + p - //must be called on initialisation - val setCallbacks : js.ThisFunction2[Parser, ParseCallback, PrefixCallback, Unit] = - (thiz: Parser, quadCallback: ParseCallback, prefixCallback: PrefixCallback) => - thiz._readCallback = thiz._readInTopContext - thiz._sparqlStyle = false - thiz._prefixes = new js.Object() - // thiz._prefixes._ = ??? - thiz._prefixCallback = prefixCallback - thiz._inversePredicate = false - thiz._quantified = new js.Object() - thiz._callback = quadCallback - val callbackFn: TokenCallback = (err: js.UndefOr[js.Error], tok: js.UndefOr[n3js.n3.mod.Token]) => - if err != null && err.isDefined then - thiz._callback(err,js.undefined,js.undefined) + // must be called on initialisation + val setCallbacks: js.ThisFunction2[Parser, ParseCallback, PrefixCallback, Unit] = + (thiz: Parser, quadCallback: ParseCallback, prefixCallback: PrefixCallback) => + thiz._readCallback = thiz._readInTopContext + thiz._sparqlStyle = false + thiz._prefixes = new js.Object() + // thiz._prefixes._ = ??? + thiz._prefixCallback = prefixCallback + thiz._inversePredicate = false + thiz._quantified = new js.Object() + thiz._callback = quadCallback + val callbackFn: TokenCallback = + (err: js.UndefOr[js.Error], tok: js.UndefOr[n3js.n3.mod.Token]) => + if err != null && err.isDefined then thiz._callback(err, js.undefined, js.undefined) // thiz._callback = noopCallback - else if tok != null && tok.isDefined then - thiz._readCallback = thiz._readCallback(thiz, tok.get).asInstanceOf[TokenFn] - else thiz._callback(new js.Error("don't have a callback to continue parsing"), js.undefined, js.undefined) - thiz._lexer.setCallback(thiz._lexer, callbackFn) - () + else if tok != null && tok.isDefined then + thiz._readCallback = thiz._readCallback(thiz, tok.get).asInstanceOf[TokenFn] + else + thiz._callback( + new js.Error("don't have a callback to continue parsing"), + js.undefined, + js.undefined + ) + thiz._lexer.setCallback(thiz._lexer, callbackFn) + () - val noopCallback: ParseCallback = - (err: js.UndefOr[js.Error], quad: js.UndefOr[Quad], pre: js.UndefOr[n3js.n3.mod.Prefixes[NamedNode]]) => () + val noopCallback: ParseCallback = ( + err: js.UndefOr[js.Error], + quad: js.UndefOr[Quad], + pre: js.UndefOr[n3js.n3.mod.Prefixes[NamedNode]] + ) => () - //this is the function f in `input.on('data', f)` - val parseChunk: js.ThisFunction2[Parser, String, Boolean, Unit] = - (thiz: Parser, chunk: String, end: Boolean) => - thiz._lexer.tokenizeChunk(chunk,end) - () - -end Parser //obj + // this is the function f in `input.on('data', f)` + val parseChunk: js.ThisFunction2[Parser, String, Boolean, Unit] = + (thiz: Parser, chunk: String, end: Boolean) => + thiz._lexer.tokenizeChunk(chunk, end) + () +end Parser //obj diff --git a/n3js/src/main/scala/n3js/n3/mod/ParserOptions.scala b/n3js/src/main/scala/n3js/n3/mod/ParserOptions.scala index aa94af7..ce39981 100644 --- a/n3js/src/main/scala/n3js/n3/mod/ParserOptions.scala +++ b/n3js/src/main/scala/n3js/n3/mod/ParserOptions.scala @@ -6,51 +6,50 @@ import scala.scalajs.js.`|` import scala.scalajs.js.annotation.{JSGlobalScope, JSGlobal, JSImport, JSName, JSBracketAccess} @js.native -trait ParserOptions extends StObject { - - var baseIRI: js.UndefOr[String] = js.native - - var blankNodePrefix: js.UndefOr[String] = js.native - - var factory: js.UndefOr[run.cosy.rdfjs.model.DataFactory] = js.native - - // string type is here to maintain backwards compatibility - consider removing when - // updating major version - var format: js.UndefOr[String | MimeFormat] = js.native -} -object ParserOptions { - - @scala.inline - def apply(): ParserOptions = { - val __obj = js.Dynamic.literal() - __obj.asInstanceOf[ParserOptions] - } - - @scala.inline - implicit class ParserOptionsMutableBuilder[Self <: ParserOptions] (val x: Self) extends AnyVal { - - @scala.inline - def setBaseIRI(value: String): Self = StObject.set(x, "baseIRI", value.asInstanceOf[js.Any]) - - @scala.inline - def setBaseIRIUndefined: Self = StObject.set(x, "baseIRI", js.undefined) - - @scala.inline - def setBlankNodePrefix(value: String): Self = StObject.set(x, "blankNodePrefix", value.asInstanceOf[js.Any]) - - @scala.inline - def setBlankNodePrefixUndefined: Self = StObject.set(x, "blankNodePrefix", js.undefined) - - @scala.inline - def setFactory(value: run.cosy.rdfjs.model.DataFactory): Self = StObject.set(x, "factory", value.asInstanceOf[js.Any]) - - @scala.inline - def setFactoryUndefined: Self = StObject.set(x, "factory", js.undefined) - - @scala.inline - def setFormat(value: String | MimeFormat): Self = StObject.set(x, "format", value.asInstanceOf[js.Any]) - - @scala.inline - def setFormatUndefined: Self = StObject.set(x, "format", js.undefined) - } -} +trait ParserOptions extends StObject: + + var baseIRI: js.UndefOr[String] = js.native + + var blankNodePrefix: js.UndefOr[String] = js.native + + var factory: js.UndefOr[run.cosy.rdfjs.model.DataFactory] = js.native + + // string type is here to maintain backwards compatibility - consider removing when + // updating major version + var format: js.UndefOr[String | MimeFormat] = js.native +object ParserOptions: + + @scala.inline + def apply(): ParserOptions = + val __obj = js.Dynamic.literal() + __obj.asInstanceOf[ParserOptions] + + @scala.inline + implicit class ParserOptionsMutableBuilder[Self <: ParserOptions](val x: Self) extends AnyVal: + + @scala.inline + def setBaseIRI(value: String): Self = StObject.set(x, "baseIRI", value.asInstanceOf[js.Any]) + + @scala.inline + def setBaseIRIUndefined: Self = StObject.set(x, "baseIRI", js.undefined) + + @scala.inline + def setBlankNodePrefix(value: String): Self = StObject + .set(x, "blankNodePrefix", value.asInstanceOf[js.Any]) + + @scala.inline + def setBlankNodePrefixUndefined: Self = StObject.set(x, "blankNodePrefix", js.undefined) + + @scala.inline + def setFactory(value: run.cosy.rdfjs.model.DataFactory): Self = StObject + .set(x, "factory", value.asInstanceOf[js.Any]) + + @scala.inline + def setFactoryUndefined: Self = StObject.set(x, "factory", js.undefined) + + @scala.inline + def setFormat(value: String | MimeFormat): Self = StObject + .set(x, "format", value.asInstanceOf[js.Any]) + + @scala.inline + def setFormatUndefined: Self = StObject.set(x, "format", js.undefined) diff --git a/n3js/src/main/scala/n3js/n3/mod/Star.scala b/n3js/src/main/scala/n3js/n3/mod/Star.scala index a581706..c4bbbb9 100644 --- a/n3js/src/main/scala/n3js/n3/mod/Star.scala +++ b/n3js/src/main/scala/n3js/n3/mod/Star.scala @@ -5,20 +5,19 @@ import scala.scalajs.js import scala.scalajs.js.`|` import scala.scalajs.js.annotation.{JSGlobalScope, JSGlobal, JSImport, JSName, JSBracketAccess} -/* Rewritten from type alias, can be one of: +/* Rewritten from type alias, can be one of: - n3js.n3.n3Strings.Asterisk - n3js.n3.n3Strings.star - n3js.n3.n3Strings.`-star` -*/ + */ trait Star extends StObject -object Star { - - @scala.inline - def `-star`: n3js.n3.n3Strings.`-star` = "-star".asInstanceOf[n3js.n3.n3Strings.`-star`] - - @scala.inline - def Asterisk: n3js.n3.n3Strings.Asterisk = "*".asInstanceOf[n3js.n3.n3Strings.Asterisk] - - @scala.inline - def star: n3js.n3.n3Strings.star = "star".asInstanceOf[n3js.n3.n3Strings.star] -} +object Star: + + @scala.inline + def `-star`: n3js.n3.n3Strings.`-star` = "-star".asInstanceOf[n3js.n3.n3Strings.`-star`] + + @scala.inline + def Asterisk: n3js.n3.n3Strings.Asterisk = "*".asInstanceOf[n3js.n3.n3Strings.Asterisk] + + @scala.inline + def star: n3js.n3.n3Strings.star = "star".asInstanceOf[n3js.n3.n3Strings.star] diff --git a/n3js/src/main/scala/n3js/n3/mod/Token.scala b/n3js/src/main/scala/n3js/n3/mod/Token.scala index 8c23f23..fd9d564 100644 --- a/n3js/src/main/scala/n3js/n3/mod/Token.scala +++ b/n3js/src/main/scala/n3js/n3/mod/Token.scala @@ -6,44 +6,40 @@ import scala.scalajs.js.`|` import scala.scalajs.js.annotation.{JSGlobalScope, JSGlobal, JSImport, JSName, JSBracketAccess} @js.native -trait Token extends StObject { - - var line: Double = js.native - - var prefix: js.UndefOr[String] = js.native - - var `type`: String = js.native - - var value: js.UndefOr[String] = js.native -} -object Token { - - @scala.inline - def apply(line: Double, `type`: String): Token = { - val __obj = js.Dynamic.literal(line = line.asInstanceOf[js.Any]) - __obj.updateDynamic("type")(`type`.asInstanceOf[js.Any]) - __obj.asInstanceOf[Token] - } - - @scala.inline - implicit class TokenMutableBuilder[Self <: Token] (val x: Self) extends AnyVal { - - @scala.inline - def setLine(value: Double): Self = StObject.set(x, "line", value.asInstanceOf[js.Any]) - - @scala.inline - def setPrefix(value: String): Self = StObject.set(x, "prefix", value.asInstanceOf[js.Any]) - - @scala.inline - def setPrefixUndefined: Self = StObject.set(x, "prefix", js.undefined) - - @scala.inline - def setType(value: String): Self = StObject.set(x, "type", value.asInstanceOf[js.Any]) - - @scala.inline - def setValue(value: String): Self = StObject.set(x, "value", value.asInstanceOf[js.Any]) - - @scala.inline - def setValueUndefined: Self = StObject.set(x, "value", js.undefined) - } -} +trait Token extends StObject: + + var line: Double = js.native + + var prefix: js.UndefOr[String] = js.native + + var `type`: String = js.native + + var value: js.UndefOr[String] = js.native +object Token: + + @scala.inline + def apply(line: Double, `type`: String): Token = + val __obj = js.Dynamic.literal(line = line.asInstanceOf[js.Any]) + __obj.updateDynamic("type")(`type`.asInstanceOf[js.Any]) + __obj.asInstanceOf[Token] + + @scala.inline + implicit class TokenMutableBuilder[Self <: Token](val x: Self) extends AnyVal: + + @scala.inline + def setLine(value: Double): Self = StObject.set(x, "line", value.asInstanceOf[js.Any]) + + @scala.inline + def setPrefix(value: String): Self = StObject.set(x, "prefix", value.asInstanceOf[js.Any]) + + @scala.inline + def setPrefixUndefined: Self = StObject.set(x, "prefix", js.undefined) + + @scala.inline + def setType(value: String): Self = StObject.set(x, "type", value.asInstanceOf[js.Any]) + + @scala.inline + def setValue(value: String): Self = StObject.set(x, "value", value.asInstanceOf[js.Any]) + + @scala.inline + def setValueUndefined: Self = StObject.set(x, "value", js.undefined) diff --git a/n3js/src/main/scala/n3js/n3/mod/package.scala b/n3js/src/main/scala/n3js/n3/mod/package.scala index fe8c19e..e4292fc 100644 --- a/n3js/src/main/scala/n3js/n3/mod/package.scala +++ b/n3js/src/main/scala/n3js/n3/mod/package.scala @@ -5,84 +5,88 @@ import scala.scalajs.js import scala.scalajs.js.annotation.{JSGlobalScope, JSGlobal, JSImport, JSName, JSBracketAccess} import run.cosy.rdfjs.model.* -package object mod { - +package object mod: + // @scala.inline // def termFromId( // id: java.lang.String, // factory: run.cosy.rdfjs.model.DataFactory -// ): n3js.n3.mod.Term = -// (n3js.n3.mod.^.asInstanceOf[js.Dynamic].applyDynamic("termFromId")(id.asInstanceOf[js.Any], +// ): n3js.n3.mod.Term = +// (n3js.n3.mod.^.asInstanceOf[js.Dynamic].applyDynamic("termFromId")(id.asInstanceOf[js.Any], // factory.asInstanceOf[js.Any])).asInstanceOf[n3js.n3.mod.Term] - - @scala.inline - def termToId(term: n3js.n3.mod.Term): java.lang.String = n3js.n3.mod.^.asInstanceOf[js.Dynamic].applyDynamic("termToId")(term.asInstanceOf[js.Any]).asInstanceOf[java.lang.String] - - type BaseFormatVariant = n3js.n3.mod.BaseFormat | n3js.std.Lowercase[n3js.n3.mod.BaseFormat] - - type ErrorCallback = js.Function2[/* err */ js.Error, /* result */ js.Any, scala.Unit] - - type MimeFormat = n3js.n3.mod.MimeSubtype | n3js.n3.n3Strings.DollarLeftcurlybracketMimeTypeRightcurlybracketSlashDollarLeftcurlybracketMimeSubtypeRightcurlybracket - - type MimeSubtype = n3js.n3.mod.BaseFormatVariant | n3js.n3.n3Strings.DollarLeftcurlybracketBaseFormatVariantRightcurlybracketDollarLeftcurlybracketStarRightcurlybracket - - type OTerm = run.cosy.rdfjs.model.Term[?] | java.lang.String | scala.Null - - type ParseCallback = js.Function3[ - /* error */ js.UndefOr[js.Error], - /* quad */ js.UndefOr[Quad], - /* prefixes */ js.UndefOr[n3js.n3.mod.Prefixes[NamedNode]], - scala.Unit - ] - - type PrefixCallback = js.Function2[ - /* prefix */ java.lang.String, - /* prefixNode */ run.cosy.rdfjs.model.NamedNode, - scala.Unit - ] - - type PrefixedToIri = js.Function1[/* suffix */ java.lang.String, run.cosy.rdfjs.model.NamedNode] - - type Prefixes[I] = org.scalablytyped.runtime.StringDictionary[I] - - type QuadCallback[Q /* <: n3js.n3.mod.BaseQuad */] = js.Function1[/* result */ Q, scala.Unit] - - /* Rewritten from type alias, can be one of: + + @scala.inline + def termToId(term: n3js.n3.mod.Term): java.lang.String = n3js.n3.mod.^.asInstanceOf[js.Dynamic] + .applyDynamic("termToId")(term.asInstanceOf[js.Any]).asInstanceOf[java.lang.String] + + type BaseFormatVariant = n3js.n3.mod.BaseFormat | n3js.std.Lowercase[n3js.n3.mod.BaseFormat] + + type ErrorCallback = js.Function2[ /* err */ js.Error, /* result */ js.Any, scala.Unit] + + type MimeFormat = n3js.n3.mod.MimeSubtype | + n3js.n3.n3Strings.DollarLeftcurlybracketMimeTypeRightcurlybracketSlashDollarLeftcurlybracketMimeSubtypeRightcurlybracket + + type MimeSubtype = n3js.n3.mod.BaseFormatVariant | + n3js.n3.n3Strings.DollarLeftcurlybracketBaseFormatVariantRightcurlybracketDollarLeftcurlybracketStarRightcurlybracket + + type OTerm = run.cosy.rdfjs.model.Term[?] | java.lang.String | scala.Null + + type ParseCallback = js.Function3[ + /* error */ js.UndefOr[js.Error], + /* quad */ js.UndefOr[Quad], + /* prefixes */ js.UndefOr[n3js.n3.mod.Prefixes[NamedNode]], + scala.Unit + ] + + type PrefixCallback = js.Function2[ + /* prefix */ java.lang.String, + /* prefixNode */ run.cosy.rdfjs.model.NamedNode, + scala.Unit + ] + + type PrefixedToIri = js.Function1[ /* suffix */ java.lang.String, run.cosy.rdfjs.model.NamedNode] + + type Prefixes[I] = org.scalablytyped.runtime.StringDictionary[I] + + type QuadCallback[Q /* <: n3js.n3.mod.BaseQuad */ ] = js.Function1[ /* result */ Q, scala.Unit] + + /* Rewritten from type alias, can be one of: - n3js.n3.mod.DefaultGraph - n3js.n3.mod.NamedNode[java.lang.String] - n3js.n3.mod.BlankNode - n3js.n3.mod.Variable - */ - type QuadGraph = run.cosy.rdfjs.model.Quad.Graph | run.cosy.rdfjs.model.Variable - - /* Rewritten from type alias, can be one of: + */ + type QuadGraph = run.cosy.rdfjs.model.Quad.Graph | run.cosy.rdfjs.model.Variable + + /* Rewritten from type alias, can be one of: - n3js.n3.mod.NamedNode[java.lang.String] - n3js.n3.mod.Literal - n3js.n3.mod.BlankNode - n3js.n3.mod.Variable - */ - type QuadObject = run.cosy.rdfjs.model.Quad.Object | run.cosy.rdfjs.model.Variable - - type QuadPredicate[Q /* <: n3js.n3.mod.BaseQuad */] = js.Function1[/* result */ Q, scala.Boolean] - - /* Rewritten from type alias, can be one of: + */ + type QuadObject = run.cosy.rdfjs.model.Quad.Object | run.cosy.rdfjs.model.Variable + + type QuadPredicate[Q /* <: n3js.n3.mod.BaseQuad */ ] = + js.Function1[ /* result */ Q, scala.Boolean] + + /* Rewritten from type alias, can be one of: - n3js.n3.mod.NamedNode[java.lang.String] - n3js.n3.mod.BlankNode - n3js.n3.mod.Variable - */ - type QuadSubject = run.cosy.rdfjs.model.Quad.Subject | run.cosy.rdfjs.model.Variable - - type Quad_Predicate = run.cosy.rdfjs.model.Quad.Predicate | run.cosy.rdfjs.model.Variable - - - /* Rewritten from type alias, can be one of: + */ + type QuadSubject = run.cosy.rdfjs.model.Quad.Subject | run.cosy.rdfjs.model.Variable + + type Quad_Predicate = run.cosy.rdfjs.model.Quad.Predicate | run.cosy.rdfjs.model.Variable + + /* Rewritten from type alias, can be one of: - n3js.n3.mod.NamedNode[java.lang.String] - n3js.n3.mod.BlankNode - n3js.n3.mod.Literal - n3js.n3.mod.Variable - n3js.n3.mod.DefaultGraph - */ - type Term = run.cosy.rdfjs.model.Term[?] | run.cosy.rdfjs.model.Variable + */ + type Term = run.cosy.rdfjs.model.Term[?] | run.cosy.rdfjs.model.Variable - type TokenCallback = js.Function2[/* error */ js.UndefOr[js.Error], /* token */ js.UndefOr[n3js.n3.mod.Token], scala.Unit] -} + type TokenCallback = js.Function2[ /* error */ js.UndefOr[js.Error], /* token */ js.UndefOr[ + n3js.n3.mod.Token + ], scala.Unit] diff --git a/n3js/src/main/scala/n3js/n3/n3Strings.scala b/n3js/src/main/scala/n3js/n3/n3Strings.scala index 2e39d0c..52cd3bc 100644 --- a/n3js/src/main/scala/n3js/n3/n3Strings.scala +++ b/n3js/src/main/scala/n3js/n3/n3Strings.scala @@ -8,148 +8,131 @@ import scala.scalajs.js import scala.scalajs.js.`|` import scala.scalajs.js.annotation.{JSGlobalScope, JSGlobal, JSImport, JSName, JSBracketAccess} -object n3Strings { - - @js.native - sealed trait `-star` - extends StObject - with Star - @scala.inline - def `-star`: `-star` = "-star".asInstanceOf[`-star`] - - @js.native - sealed trait Asterisk - extends StObject - with Star - @scala.inline - def Asterisk: Asterisk = "*".asInstanceOf[Asterisk] - - @js.native - sealed trait BlankNode extends StObject - @scala.inline - def BlankNode: BlankNode = "BlankNode".asInstanceOf[BlankNode] - - @js.native - sealed trait DefaultGraph extends StObject - @scala.inline - def DefaultGraph: DefaultGraph = "DefaultGraph".asInstanceOf[DefaultGraph] - - @js.native - sealed trait DollarLeftcurlybracketBaseFormatVariantRightcurlybracketDollarLeftcurlybracketStarRightcurlybracket extends StObject - @scala.inline - def DollarLeftcurlybracketBaseFormatVariantRightcurlybracketDollarLeftcurlybracketStarRightcurlybracket: DollarLeftcurlybracketBaseFormatVariantRightcurlybracketDollarLeftcurlybracketStarRightcurlybracket = "${BaseFormatVariant}${Star}".asInstanceOf[DollarLeftcurlybracketBaseFormatVariantRightcurlybracketDollarLeftcurlybracketStarRightcurlybracket] - - @js.native - sealed trait DollarLeftcurlybracketMimeTypeRightcurlybracketSlashDollarLeftcurlybracketMimeSubtypeRightcurlybracket extends StObject - @scala.inline - def DollarLeftcurlybracketMimeTypeRightcurlybracketSlashDollarLeftcurlybracketMimeSubtypeRightcurlybracket: DollarLeftcurlybracketMimeTypeRightcurlybracketSlashDollarLeftcurlybracketMimeSubtypeRightcurlybracket = "${MimeType}/${MimeSubtype}".asInstanceOf[DollarLeftcurlybracketMimeTypeRightcurlybracketSlashDollarLeftcurlybracketMimeSubtypeRightcurlybracket] - - @js.native - sealed trait Literal extends StObject - @scala.inline - def Literal: Literal = "Literal".asInstanceOf[Literal] - - @js.native - sealed trait `N-Quads` - extends StObject - with BaseFormat - @scala.inline - def `N-Quads`: `N-Quads` = "N-Quads".asInstanceOf[`N-Quads`] - - @js.native - sealed trait `N-Triples` - extends StObject - with BaseFormat - @scala.inline - def `N-Triples`: `N-Triples` = "N-Triples".asInstanceOf[`N-Triples`] - - @js.native - sealed trait N3 - extends StObject - with BaseFormat - @scala.inline - def N3: N3 = "N3".asInstanceOf[N3] - - @js.native - sealed trait NamedNode extends StObject - @scala.inline - def NamedNode: NamedNode = "NamedNode".asInstanceOf[NamedNode] - - @js.native - sealed trait Notation3 - extends StObject - with BaseFormat - @scala.inline - def Notation3: Notation3 = "Notation3".asInstanceOf[Notation3] - - @js.native - sealed trait Quad extends StObject - @scala.inline - def Quad: Quad = "Quad".asInstanceOf[Quad] - - @js.native - sealed trait TriG - extends StObject - with BaseFormat - @scala.inline - def TriG: TriG = "TriG".asInstanceOf[TriG] - - @js.native - sealed trait Turtle - extends StObject - with BaseFormat - @scala.inline - def Turtle: Turtle = "Turtle".asInstanceOf[Turtle] - - @js.native - sealed trait Variable extends StObject - @scala.inline - def Variable: Variable = "Variable".asInstanceOf[Variable] - - @js.native - sealed trait _empty extends StObject - @scala.inline - def _empty: _empty = "".asInstanceOf[_empty] - - @js.native - sealed trait application - extends StObject - with MimeType - @scala.inline - def application: application = "application".asInstanceOf[application] - - @js.native - sealed trait example - extends StObject - with MimeType - @scala.inline - def example: example = "example".asInstanceOf[example] - - @js.native - sealed trait message - extends StObject - with MimeType - @scala.inline - def message: message = "message".asInstanceOf[message] - - @js.native - sealed trait multipart - extends StObject - with MimeType - @scala.inline - def multipart: multipart = "multipart".asInstanceOf[multipart] - - @js.native - sealed trait star - extends StObject - with Star - @scala.inline - def star: star = "star".asInstanceOf[star] - - @js.native - sealed trait text - extends StObject - with MimeType - @scala.inline - def text: text = "text".asInstanceOf[text] -} +object n3Strings: + + @js.native + sealed trait `-star` extends StObject with Star + @scala.inline + def `-star`: `-star` = "-star".asInstanceOf[`-star`] + + @js.native + sealed trait Asterisk extends StObject with Star + @scala.inline + def Asterisk: Asterisk = "*".asInstanceOf[Asterisk] + + @js.native + sealed trait BlankNode extends StObject + @scala.inline + def BlankNode: BlankNode = "BlankNode".asInstanceOf[BlankNode] + + @js.native + sealed trait DefaultGraph extends StObject + @scala.inline + def DefaultGraph: DefaultGraph = "DefaultGraph".asInstanceOf[DefaultGraph] + + @js.native + sealed trait DollarLeftcurlybracketBaseFormatVariantRightcurlybracketDollarLeftcurlybracketStarRightcurlybracket + extends StObject + @scala.inline + def DollarLeftcurlybracketBaseFormatVariantRightcurlybracketDollarLeftcurlybracketStarRightcurlybracket + : DollarLeftcurlybracketBaseFormatVariantRightcurlybracketDollarLeftcurlybracketStarRightcurlybracket = + "${BaseFormatVariant}${Star}" + .asInstanceOf[ + DollarLeftcurlybracketBaseFormatVariantRightcurlybracketDollarLeftcurlybracketStarRightcurlybracket + ] + + @js.native + sealed trait DollarLeftcurlybracketMimeTypeRightcurlybracketSlashDollarLeftcurlybracketMimeSubtypeRightcurlybracket + extends StObject + @scala.inline + def DollarLeftcurlybracketMimeTypeRightcurlybracketSlashDollarLeftcurlybracketMimeSubtypeRightcurlybracket + : DollarLeftcurlybracketMimeTypeRightcurlybracketSlashDollarLeftcurlybracketMimeSubtypeRightcurlybracket = + "${MimeType}/${MimeSubtype}" + .asInstanceOf[ + DollarLeftcurlybracketMimeTypeRightcurlybracketSlashDollarLeftcurlybracketMimeSubtypeRightcurlybracket + ] + + @js.native + sealed trait Literal extends StObject + @scala.inline + def Literal: Literal = "Literal".asInstanceOf[Literal] + + @js.native + sealed trait `N-Quads` extends StObject with BaseFormat + @scala.inline + def `N-Quads`: `N-Quads` = "N-Quads".asInstanceOf[`N-Quads`] + + @js.native + sealed trait `N-Triples` extends StObject with BaseFormat + @scala.inline + def `N-Triples`: `N-Triples` = "N-Triples".asInstanceOf[`N-Triples`] + + @js.native + sealed trait N3 extends StObject with BaseFormat + @scala.inline + def N3: N3 = "N3".asInstanceOf[N3] + + @js.native + sealed trait NamedNode extends StObject + @scala.inline + def NamedNode: NamedNode = "NamedNode".asInstanceOf[NamedNode] + + @js.native + sealed trait Notation3 extends StObject with BaseFormat + @scala.inline + def Notation3: Notation3 = "Notation3".asInstanceOf[Notation3] + + @js.native + sealed trait Quad extends StObject + @scala.inline + def Quad: Quad = "Quad".asInstanceOf[Quad] + + @js.native + sealed trait TriG extends StObject with BaseFormat + @scala.inline + def TriG: TriG = "TriG".asInstanceOf[TriG] + + @js.native + sealed trait Turtle extends StObject with BaseFormat + @scala.inline + def Turtle: Turtle = "Turtle".asInstanceOf[Turtle] + + @js.native + sealed trait Variable extends StObject + @scala.inline + def Variable: Variable = "Variable".asInstanceOf[Variable] + + @js.native + sealed trait _empty extends StObject + @scala.inline + def _empty: _empty = "".asInstanceOf[_empty] + + @js.native + sealed trait application extends StObject with MimeType + @scala.inline + def application: application = "application".asInstanceOf[application] + + @js.native + sealed trait example extends StObject with MimeType + @scala.inline + def example: example = "example".asInstanceOf[example] + + @js.native + sealed trait message extends StObject with MimeType + @scala.inline + def message: message = "message".asInstanceOf[message] + + @js.native + sealed trait multipart extends StObject with MimeType + @scala.inline + def multipart: multipart = "multipart".asInstanceOf[multipart] + + @js.native + sealed trait star extends StObject with Star + @scala.inline + def star: star = "star".asInstanceOf[star] + + @js.native + sealed trait text extends StObject with MimeType + @scala.inline + def text: text = "text".asInstanceOf[text] diff --git a/n3js/src/main/scala/n3js/node/eventsMod.scala b/n3js/src/main/scala/n3js/node/eventsMod.scala index d118270..e4e99bf 100644 --- a/n3js/src/main/scala/n3js/node/eventsMod.scala +++ b/n3js/src/main/scala/n3js/node/eventsMod.scala @@ -7,7 +7,7 @@ // import scala.scalajs.js.annotation.{JSGlobalScope, JSGlobal, JSImport, JSName, JSBracketAccess} // object eventsMod { - + // /** // * The `EventEmitter` class is defined and exposed by the `events` module: // * @@ -29,113 +29,113 @@ // def this(options: EventEmitterOptions) = this() // } // object EventEmitter - + // @js.native // trait Abortable extends StObject { - + // /** // * When provided the corresponding `AbortController` can be used to cancel an asynchronous action. // */ // var signal: js.UndefOr[AbortSignal] = js.native // } // object Abortable { - + // @scala.inline // def apply(): Abortable = { // val __obj = js.Dynamic.literal() // __obj.asInstanceOf[Abortable] // } - + // @scala.inline // implicit class AbortableMutableBuilder[Self <: Abortable] (val x: Self) extends AnyVal { - + // @scala.inline // def setSignal(value: AbortSignal): Self = StObject.set(x, "signal", value.asInstanceOf[js.Any]) - + // @scala.inline // def setSignalUndefined: Self = StObject.set(x, "signal", js.undefined) // } // } - + // @js.native // trait DOMEventTarget extends StObject { - + // def addEventListener(eventName: String, listener: js.Function1[/* repeated */ js.Any, Unit]): js.Any = js.native // def addEventListener(eventName: String, listener: js.Function1[/* repeated */ js.Any, Unit], opts: Once): js.Any = js.native // } - + // @js.native // trait EventEmitterOptions extends StObject { - + // /** // * Enables automatic capturing of promise rejection. // */ // var captureRejections: js.UndefOr[Boolean] = js.native // } // object EventEmitterOptions { - + // @scala.inline // def apply(): EventEmitterOptions = { // val __obj = js.Dynamic.literal() // __obj.asInstanceOf[EventEmitterOptions] // } - + // @scala.inline // implicit class EventEmitterOptionsMutableBuilder[Self <: EventEmitterOptions] (val x: Self) extends AnyVal { - + // @scala.inline // def setCaptureRejections(value: Boolean): Self = StObject.set(x, "captureRejections", value.asInstanceOf[js.Any]) - + // @scala.inline // def setCaptureRejectionsUndefined: Self = StObject.set(x, "captureRejections", js.undefined) // } // } - + // @js.native // trait NodeEventTarget extends StObject { - + // def once(eventName: String, listener: js.Function1[/* repeated */ js.Any, Unit]): this.type = js.native // def once(eventName: js.Symbol, listener: js.Function1[/* repeated */ js.Any, Unit]): this.type = js.native // } - + // @js.native // trait StaticEventEmitterOptions extends StObject { - + // var signal: js.UndefOr[AbortSignal] = js.native // } // object StaticEventEmitterOptions { - + // @scala.inline // def apply(): StaticEventEmitterOptions = { // val __obj = js.Dynamic.literal() // __obj.asInstanceOf[StaticEventEmitterOptions] // } - + // @scala.inline // implicit class StaticEventEmitterOptionsMutableBuilder[Self <: StaticEventEmitterOptions] (val x: Self) extends AnyVal { - + // @scala.inline // def setSignal(value: AbortSignal): Self = StObject.set(x, "signal", value.asInstanceOf[js.Any]) - + // @scala.inline // def setSignalUndefined: Self = StObject.set(x, "signal", js.undefined) // } // } - + // object global { - + // object NodeJS { - + // @js.native // trait EventEmitter extends StObject { - + // /** // * Alias for `emitter.on(eventName, listener)`. // * @since v0.1.26 // */ // def addListener(eventName: String, listener: js.Function1[/* repeated */ js.Any, Unit]): this.type = js.native // def addListener(eventName: js.Symbol, listener: js.Function1[/* repeated */ js.Any, Unit]): this.type = js.native - + // /** // * Synchronously calls each of the listeners registered for the event named`eventName`, in the order they were registered, passing the supplied arguments // * to each. @@ -178,7 +178,7 @@ // */ // def emit(eventName: String, args: js.Any*): Boolean = js.native // def emit(eventName: js.Symbol, args: js.Any*): Boolean = js.native - + // /** // * Returns an array listing the events for which the emitter has registered // * listeners. The values in the array are strings or `Symbol`s. @@ -198,14 +198,14 @@ // * @since v6.0.0 // */ // def eventNames(): js.Array[String | js.Symbol] = js.native - + // /** // * Returns the current max listener value for the `EventEmitter` which is either // * set by `emitter.setMaxListeners(n)` or defaults to {@link defaultMaxListeners}. // * @since v1.0.0 // */ // def getMaxListeners(): Double = js.native - + // /** // * Returns the number of listeners listening to the event named `eventName`. // * @since v3.2.0 @@ -213,7 +213,7 @@ // */ // def listenerCount(eventName: String): Double = js.native // def listenerCount(eventName: js.Symbol): Double = js.native - + // /** // * Returns a copy of the array of listeners for the event named `eventName`. // * @@ -228,14 +228,14 @@ // */ // def listeners(eventName: String): js.Array[js.Function] = js.native // def listeners(eventName: js.Symbol): js.Array[js.Function] = js.native - + // /** // * Alias for `emitter.removeListener()`. // * @since v10.0.0 // */ // def off(eventName: String, listener: js.Function1[/* repeated */ js.Any, Unit]): this.type = js.native // def off(eventName: js.Symbol, listener: js.Function1[/* repeated */ js.Any, Unit]): this.type = js.native - + // /** // * Adds the `listener` function to the end of the listeners array for the // * event named `eventName`. No checks are made to see if the `listener` has @@ -268,7 +268,7 @@ // */ // def on(eventName: String, listener: js.Function1[/* repeated */ js.Any, Unit]): this.type = js.native // def on(eventName: js.Symbol, listener: js.Function1[/* repeated */ js.Any, Unit]): this.type = js.native - + // /** // * Adds a **one-time**`listener` function for the event named `eventName`. The // * next time `eventName` is triggered, this listener is removed and then invoked. @@ -299,7 +299,7 @@ // */ // def once(eventName: String, listener: js.Function1[/* repeated */ js.Any, Unit]): this.type = js.native // def once(eventName: js.Symbol, listener: js.Function1[/* repeated */ js.Any, Unit]): this.type = js.native - + // /** // * Adds the `listener` function to the _beginning_ of the listeners array for the // * event named `eventName`. No checks are made to see if the `listener` has @@ -319,7 +319,7 @@ // */ // def prependListener(eventName: String, listener: js.Function1[/* repeated */ js.Any, Unit]): this.type = js.native // def prependListener(eventName: js.Symbol, listener: js.Function1[/* repeated */ js.Any, Unit]): this.type = js.native - + // /** // * Adds a **one-time**`listener` function for the event named `eventName` to the_beginning_ of the listeners array. The next time `eventName` is triggered, this // * listener is removed, and then invoked. @@ -337,7 +337,7 @@ // */ // def prependOnceListener(eventName: String, listener: js.Function1[/* repeated */ js.Any, Unit]): this.type = js.native // def prependOnceListener(eventName: js.Symbol, listener: js.Function1[/* repeated */ js.Any, Unit]): this.type = js.native - + // /** // * Returns a copy of the array of listeners for the event named `eventName`, // * including any wrappers (such as those created by `.once()`). @@ -369,7 +369,7 @@ // */ // def rawListeners(eventName: String): js.Array[js.Function] = js.native // def rawListeners(eventName: js.Symbol): js.Array[js.Function] = js.native - + // /** // * Removes all listeners, or those of the specified `eventName`. // * @@ -383,7 +383,7 @@ // def removeAllListeners(): this.type = js.native // def removeAllListeners(event: String): this.type = js.native // def removeAllListeners(event: js.Symbol): this.type = js.native - + // /** // * Removes the specified `listener` from the listener array for the event named`eventName`. // * @@ -465,7 +465,7 @@ // */ // def removeListener(eventName: String, listener: js.Function1[/* repeated */ js.Any, Unit]): this.type = js.native // def removeListener(eventName: js.Symbol, listener: js.Function1[/* repeated */ js.Any, Unit]): this.type = js.native - + // /** // * By default `EventEmitter`s will print a warning if more than `10` listeners are // * added for a particular event. This is a useful default that helps finding diff --git a/n3js/src/main/scala/n3js/std/ArrayLike.scala b/n3js/src/main/scala/n3js/std/ArrayLike.scala index 455afd0..ee23333 100644 --- a/n3js/src/main/scala/n3js/std/ArrayLike.scala +++ b/n3js/src/main/scala/n3js/std/ArrayLike.scala @@ -7,24 +7,19 @@ import scala.scalajs.js.`|` import scala.scalajs.js.annotation.{JSGlobalScope, JSGlobal, JSImport, JSName, JSBracketAccess} @js.native -trait ArrayLike[T] - extends StObject - with /* n */ NumberDictionary[T] { - - val length: Double = js.native -} -object ArrayLike { - - @scala.inline - def apply[T](length: Double): ArrayLike[T] = { - val __obj = js.Dynamic.literal(length = length.asInstanceOf[js.Any]) - __obj.asInstanceOf[ArrayLike[T]] - } - - @scala.inline - implicit class ArrayLikeMutableBuilder[Self <: ArrayLike[?], T] (val x: Self & ArrayLike[T]) extends AnyVal { - - @scala.inline - def setLength(value: Double): Self = StObject.set(x, "length", value.asInstanceOf[js.Any]) - } -} +trait ArrayLike[T] extends StObject with /* n */ NumberDictionary[T]: + + val length: Double = js.native +object ArrayLike: + + @scala.inline + def apply[T](length: Double): ArrayLike[T] = + val __obj = js.Dynamic.literal(length = length.asInstanceOf[js.Any]) + __obj.asInstanceOf[ArrayLike[T]] + + @scala.inline + implicit class ArrayLikeMutableBuilder[Self <: ArrayLike[?], T](val x: Self & ArrayLike[T]) + extends AnyVal: + + @scala.inline + def setLength(value: Double): Self = StObject.set(x, "length", value.asInstanceOf[js.Any]) diff --git a/n3js/src/main/scala/n3js/std/Iterable.scala b/n3js/src/main/scala/n3js/std/Iterable.scala index 9dd61f4..fa028ee 100644 --- a/n3js/src/main/scala/n3js/std/Iterable.scala +++ b/n3js/src/main/scala/n3js/std/Iterable.scala @@ -6,8 +6,7 @@ import scala.scalajs.js.`|` import scala.scalajs.js.annotation.{JSGlobalScope, JSGlobal, JSImport, JSName, JSBracketAccess} @js.native -trait Iterable[T] extends StObject { - - @JSName(js.Symbol.iterator) - var iterator: js.Function0[js.Iterator[T]] = js.native -} +trait Iterable[T] extends StObject: + + @JSName(js.Symbol.iterator) + var iterator: js.Function0[js.Iterator[T]] = js.native diff --git a/n3js/src/main/scala/n3js/std/package.scala b/n3js/src/main/scala/n3js/std/package.scala index 9bdb379..47dc5ef 100644 --- a/n3js/src/main/scala/n3js/std/package.scala +++ b/n3js/src/main/scala/n3js/std/package.scala @@ -5,15 +5,16 @@ import scala.scalajs.js import scala.scalajs.js.`|` import scala.scalajs.js.annotation.{JSGlobalScope, JSGlobal, JSImport, JSName, JSBracketAccess} -package object std { - - /** - * Convert string literal type to lowercase - */ - type Lowercase[S /* <: java.lang.String */] = /* import warning: transforms.QualifyReferences#resolveTypeRef many Couldn't qualify intrinsic */ js.Any - - /** - * Construct a type with a set of properties K of type T - */ - type Record[K /* <: /* keyof any */ java.lang.String */, T] = org.scalablytyped.runtime.StringDictionary[T] -} +package object std: + + /** Convert string literal type to lowercase + */ + type Lowercase[ + S /* <: java.lang.String */ + ] = /* import warning: transforms.QualifyReferences#resolveTypeRef many Couldn't qualify intrinsic */ + js.Any + + /** Construct a type with a set of properties K of type T + */ + type Record[K /* <: /* keyof any */ java.lang.String */, T] = + org.scalablytyped.runtime.StringDictionary[T] diff --git a/n3js/src/main/scala/n3js/std/stdBooleans.scala b/n3js/src/main/scala/n3js/std/stdBooleans.scala index 3d25f24..7652058 100644 --- a/n3js/src/main/scala/n3js/std/stdBooleans.scala +++ b/n3js/src/main/scala/n3js/std/stdBooleans.scala @@ -5,15 +5,14 @@ import scala.scalajs.js import scala.scalajs.js.`|` import scala.scalajs.js.annotation.{JSGlobalScope, JSGlobal, JSImport, JSName, JSBracketAccess} -object stdBooleans { - - @js.native - sealed trait `false` extends StObject - @scala.inline - def `false`: `false` = false.asInstanceOf[`false`] - - @js.native - sealed trait `true` extends StObject - @scala.inline - def `true`: `true` = true.asInstanceOf[`true`] -} +object stdBooleans: + + @js.native + sealed trait `false` extends StObject + @scala.inline + def `false`: `false` = false.asInstanceOf[`false`] + + @js.native + sealed trait `true` extends StObject + @scala.inline + def `true`: `true` = true.asInstanceOf[`true`] diff --git a/n3js/src/main/scala/n3js/std/stdStrings.scala b/n3js/src/main/scala/n3js/std/stdStrings.scala index 372f9d5..bc4c3ba 100644 --- a/n3js/src/main/scala/n3js/std/stdStrings.scala +++ b/n3js/src/main/scala/n3js/std/stdStrings.scala @@ -5,10 +5,9 @@ import scala.scalajs.js import scala.scalajs.js.`|` import scala.scalajs.js.annotation.{JSGlobalScope, JSGlobal, JSImport, JSName, JSBracketAccess} -object stdStrings { - - @js.native - sealed trait Uint8Array extends StObject - @scala.inline - def Uint8Array: Uint8Array = "Uint8Array".asInstanceOf[Uint8Array] -} +object stdStrings: + + @js.native + sealed trait Uint8Array extends StObject + @scala.inline + def Uint8Array: Uint8Array = "Uint8Array".asInstanceOf[Uint8Array] diff --git a/n3js/src/main/scala/run/cosy/app/io/n3/N3Parser.scala b/n3js/src/main/scala/run/cosy/app/io/n3/N3Parser.scala index 0a1390b..6705dfb 100644 --- a/n3js/src/main/scala/run/cosy/app/io/n3/N3Parser.scala +++ b/n3js/src/main/scala/run/cosy/app/io/n3/N3Parser.scala @@ -6,53 +6,49 @@ import run.cosy.rdfjs.model.{NamedNode, Quad} import scala.scalajs.js -object N3Parser { - def parse[F[_]](strs: Stream[F, String]): Stream[F, Quad] = - val state = ParserState() - state.parser.setCallbacks(state.parser, state.callback, noopPrefixCallback) - strs.scanChunksOpt[ParserState, String, Quad](state) { state => - if state.terminated then None - else - Some { chunk => - chunk.foreach { str => - state.parser.parseChunk(str, false) - } - (state, state.takeQuads()) - } - } - - val noopPrefixCallback: PrefixCallback = - (pre: String, node: run.cosy.rdfjs.model.NamedNode) => - println(s"received prefix $pre: <$node>") - () - - class ParserState private(val parser: n3js.n3.mod.Parser): - var terminated: Boolean = false - private var quads: List[Quad] = List() - - def takeQuads(): Chunk[Quad] = - var chunk = Chunk(quads *) - quads = List() - chunk - - val callback: ParseCallback = ( - err: js.UndefOr[js.Error], - quad: js.UndefOr[Quad], - prefixes: js.UndefOr[n3js.n3.mod.Prefixes[NamedNode]] - ) => - if err != null && err.isDefined then terminated = true - else if quad != null && quad.isDefined then - quads = quad.get :: quads - else if prefixes != null && prefixes.isDefined then terminated = true - //else terminated = true ? - - end ParserState //class - - object ParserState: - def apply(): ParserState = - val df = run.cosy.rdfjs.model.DataFactory() - val opt: ParserOptions = ParserOptions().setFactory(df) - val ps = new ParserState(n3js.n3.mod.Parser(opt)) - ps - -} +object N3Parser: + def parse[F[_]](strs: Stream[F, String]): Stream[F, Quad] = + val state = ParserState() + state.parser.setCallbacks(state.parser, state.callback, noopPrefixCallback) + strs.scanChunksOpt[ParserState, String, Quad](state) { state => + if state.terminated then None + else + Some { chunk => + chunk.foreach { str => + state.parser.parseChunk(str, false) + } + (state, state.takeQuads()) + } + } + + val noopPrefixCallback: PrefixCallback = (pre: String, node: run.cosy.rdfjs.model.NamedNode) => + println(s"received prefix $pre: <$node>") + () + + class ParserState private (val parser: n3js.n3.mod.Parser): + var terminated: Boolean = false + private var quads: List[Quad] = List() + + def takeQuads(): Chunk[Quad] = + var chunk = Chunk(quads*) + quads = List() + chunk + + val callback: ParseCallback = ( + err: js.UndefOr[js.Error], + quad: js.UndefOr[Quad], + prefixes: js.UndefOr[n3js.n3.mod.Prefixes[NamedNode]] + ) => + if err != null && err.isDefined then terminated = true + else if quad != null && quad.isDefined then quads = quad.get :: quads + else if prefixes != null && prefixes.isDefined then terminated = true + // else terminated = true ? + + end ParserState // class + + object ParserState: + def apply(): ParserState = + val df = run.cosy.rdfjs.model.DataFactory() + val opt: ParserOptions = ParserOptions().setFactory(df) + val ps = new ParserState(n3js.n3.mod.Parser(opt)) + ps diff --git a/n3js/src/test/scala/run/cosy/solid/app/io/http/N3ParserTests.scala b/n3js/src/test/scala/run/cosy/solid/app/io/http/N3ParserTests.scala index 167c062..f53a22a 100644 --- a/n3js/src/test/scala/run/cosy/solid/app/io/http/N3ParserTests.scala +++ b/n3js/src/test/scala/run/cosy/solid/app/io/http/N3ParserTests.scala @@ -9,50 +9,48 @@ import run.cosy.rdfjs.model.Quad import scala.scalajs.js -class N3ParserTests extends munit.FunSuite { - - def bbl(tag: String): String = "https://bblfish.net/" - - def w3c(tag: String): String = "https://w3.org/" - - def foaf(tag: String): String = "https://xmlns.com/foaf/0.1/" - - val ntriples1: String = - s"""<${bbl("#me")}> <${foaf("knows")}> <${w3c("timbl/card#i")}> . - |<${bbl("#me")}> <${foaf("name")}> "Henry Story" . - |""".stripMargin - val ntriples2: String = - s"""<${w3c("timbl/card#i")}> <${foaf("knows")}> <${bbl("#me")}> . - |<${bbl("timbl/card#i")}> <${foaf("name")}> "Tim Berners-Lee"@en . - |""".stripMargin - - test("Simple NTriples line Chunk Test") { - val bigNTriples = ntriples1 + ntriples2 - - //parse one chunk with 1 string - assertEquals(bigNTriples.length, 291) - val statementsStream: Stream[Pure, Quad] = N3Parser.parse(Stream(bigNTriples)) - val stats: Set[Quad] = statementsStream.toList.toSet - //no bnodes, so we can do simple set equality - assertEquals(stats.size, 4) - - //parse one chunk of two strings - val chunkedStatements: Stream[Pure, Quad] = N3Parser.parse(Stream(ntriples1, ntriples2)) - val chkdSt = chunkedStatements.toList.toSet - assertEquals(chkdSt.size, 4) - assertEquals(stats, chkdSt) - - //parse many chunks of many strings - //split the doc into strings of max 11 chars. - val chunkedStream: Stream[Pure, String] = Stream(bigNTriples.sliding(11, 11).toList *) - //partition those strings into chunks of three - val streamOfChunkStr: Stream[Pure, String] = chunkedStream.chunkN(3).flatMap(cs => Stream.chunk(cs)) - assertEquals(streamOfChunkStr.toList.size, 27, "Total number of strings") - assertEquals(streamOfChunkStr.chunks.toList.size, 9, "Total number of chunks") - - val buStream: Stream[Pure, Quad] = N3Parser.parse(streamOfChunkStr) - val bs = buStream.toList.toSet - assertEquals(bs.size, 4) - assertEquals(bs, stats) - } -} +class N3ParserTests extends munit.FunSuite: + + def bbl(tag: String): String = "https://bblfish.net/" + + def w3c(tag: String): String = "https://w3.org/" + + def foaf(tag: String): String = "https://xmlns.com/foaf/0.1/" + + val ntriples1: String = s"""<${bbl("#me")}> <${foaf("knows")}> <${w3c("timbl/card#i")}> . + |<${bbl("#me")}> <${foaf("name")}> "Henry Story" . + |""".stripMargin + val ntriples2: String = s"""<${w3c("timbl/card#i")}> <${foaf("knows")}> <${bbl("#me")}> . + |<${bbl("timbl/card#i")}> <${foaf("name")}> "Tim Berners-Lee"@en . + |""".stripMargin + + test("Simple NTriples line Chunk Test") { + val bigNTriples = ntriples1 + ntriples2 + + // parse one chunk with 1 string + assertEquals(bigNTriples.length, 291) + val statementsStream: Stream[Pure, Quad] = N3Parser.parse(Stream(bigNTriples)) + val stats: Set[Quad] = statementsStream.toList.toSet + // no bnodes, so we can do simple set equality + assertEquals(stats.size, 4) + + // parse one chunk of two strings + val chunkedStatements: Stream[Pure, Quad] = N3Parser.parse(Stream(ntriples1, ntriples2)) + val chkdSt = chunkedStatements.toList.toSet + assertEquals(chkdSt.size, 4) + assertEquals(stats, chkdSt) + + // parse many chunks of many strings + // split the doc into strings of max 11 chars. + val chunkedStream: Stream[Pure, String] = Stream(bigNTriples.sliding(11, 11).toList*) + // partition those strings into chunks of three + val streamOfChunkStr: Stream[Pure, String] = chunkedStream.chunkN(3) + .flatMap(cs => Stream.chunk(cs)) + assertEquals(streamOfChunkStr.toList.size, 27, "Total number of strings") + assertEquals(streamOfChunkStr.chunks.toList.size, 9, "Total number of chunks") + + val buStream: Stream[Pure, Quad] = N3Parser.parse(streamOfChunkStr) + val bs = buStream.toList.toSet + assertEquals(bs.size, 4) + assertEquals(bs, stats) + } diff --git a/scripts/shared/src/main/scala/scripts/AnHttpSigClient.scala b/scripts/shared/src/main/scala/scripts/AnHttpSigClient.scala index e917ef0..d4a67f4 100644 --- a/scripts/shared/src/main/scala/scripts/AnHttpSigClient.scala +++ b/scripts/shared/src/main/scala/scripts/AnHttpSigClient.scala @@ -104,7 +104,7 @@ object AnHttpSigClient: } @main - def fetch(uriStr: String = "http://localhost:8080/protected/README"): Unit = + def fetch(uriStr: String = "http://localhost:8080/protected/README"): Unit = // ioStr(uri"http://localhost:8080/").unsafeRunSync() // ioStr(uri"http://localhost:8080/protected/").unsafeRunSync() val result = emberAuthClient.flatMap { client => diff --git a/scripts/shared/src/main/scala/scripts/PemToJWT.sc b/scripts/shared/src/main/scala/scripts/PemToJWT.sc index 3c837f1..0cff674 100644 --- a/scripts/shared/src/main/scala/scripts/PemToJWT.sc +++ b/scripts/shared/src/main/scala/scripts/PemToJWT.sc @@ -45,4 +45,3 @@ println("---pub key") println(JWK.parseFromPEMEncodedObjects(pssPemPub)) println("---private key") println(JWK.parseFromPEMEncodedObjects(pssPemPriv)) - diff --git a/wallet/shared/src/main/scala/net/bblfish/app/Wallet.scala b/wallet/shared/src/main/scala/net/bblfish/app/Wallet.scala index 7d12b44..0a72f78 100644 --- a/wallet/shared/src/main/scala/net/bblfish/app/Wallet.scala +++ b/wallet/shared/src/main/scala/net/bblfish/app/Wallet.scala @@ -33,6 +33,7 @@ trait Wallet[F[_]]: /** previous requests to a server will return acls and methods that can be assumed to be valid * @param req - * @return a request with a signature if possible, otherwise an error that can be ignored + * @return + * a request with a signature if possible, otherwise an error that can be ignored */ - def signFromDB(req: Request[F]): F[Either[Throwable,Request[F]]] + def signFromDB(req: Request[F]): F[Either[Throwable, Request[F]]] diff --git a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala index 7264990..1f3856e 100644 --- a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala +++ b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala @@ -459,9 +459,9 @@ class BasicWallet[F[_], Rdf <: RDF]( result.use(fc.pure) end httpSigChallenge - /** sign the request with the first key that matches the rules in the aclGraph. (We pass the requestUrl - * too to avoid recaculating it from the request...) - */ + /** sign the request with the first key that matches the rules in the aclGraph. (We pass the + * requestUrl too to avoid recaculating it from the request...) + */ def signRequest( originalRequest: h4s.Request[F], originalRequestUrl: ll.AbsoluteUrl @@ -503,7 +503,7 @@ class BasicWallet[F[_], Rdf <: RDF]( .addHeader[Http.Request[H4]](signedReq)("Authorization", "HttpSig proof=sig1") h4ReqToHttpReq(res) Resource.liftK(x) - end signRequest + end signRequest /** This is different from middleware such as FollowRedirects, as that essentially continues the * request. Here we need to stop the request and make new ones to find the access control rules @@ -535,15 +535,15 @@ class BasicWallet[F[_], Rdf <: RDF]( case _ => ??? // fail end sign - override def signFromDB(req: h4s.Request[F]): F[Either[Throwable,h4s.Request[F]]] = + override def signFromDB(req: h4s.Request[F]): F[Either[Throwable, h4s.Request[F]]] = // todo: the DB needs to keep track of what WWW-Authenticate methods the server allows. // These will be difficult to find in the headers, as the 401 in which they appeared may be // somewhere completely different. // I will assume that the server can do HTTP-Sig for the moment // todo: fix!!! // todo: as we endup with F[Request] do we need Resources everywhere here? - findCachedAclFor(req.uri.toLL.toAbsoluteUrl, 100).flatMap { + findCachedAclFor(req.uri.toLL.toAbsoluteUrl, 100).flatMap { signRequest(req, req.uri.toLL.toAbsoluteUrl) }.attempt.use(fc.pure) - + end BasicWallet From bde97487078875f75414740a336654d5fc805913 Mon Sep 17 00:00:00 2001 From: Henry Story Date: Mon, 5 Jun 2023 08:04:26 +0200 Subject: [PATCH 40/42] scaladoc format improvements --- .scalafmt.conf | 8 ++++++++ .../scala/io/chrisdavenport/mules/http4s/CacheType.scala | 3 +-- .../src/main/scala/run/cosy/http/cache/DirTree.scala | 6 ++---- n3js/src/main/scala/n3js/std/package.scala | 6 ++---- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.scalafmt.conf b/.scalafmt.conf index 701a6c7..b53736a 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -20,6 +20,14 @@ newlines { selectChains = fold beforeMultiline = fold } +comments.wrapSingleLineMlcAsSlc = false +docstrings{ + wrap = "no" + oneline = fold + style = SpaceAsterisk +} + + fileOverride { "glob:**.sbt" { runner.dialect = scala212source3 diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheType.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheType.scala index 75b3b0d..f0035c8 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheType.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheType.scala @@ -23,8 +23,7 @@ package io.chrisdavenport.mules.http4s * Cache-Control: private, whereas public caches are not allowed to cache that information */ sealed trait CacheType: - /** Whether or not a Cache is Shared, public caches are shared, private caches are not - */ + /** Whether or not a Cache is Shared, public caches are shared, private caches are not */ def isShared: Boolean = this match case CacheType.Private => false case CacheType.Public => true diff --git a/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala b/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala index 7a5e0a4..3e13b03 100644 --- a/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala +++ b/cache/shared/src/main/scala/run/cosy/http/cache/DirTree.scala @@ -59,8 +59,7 @@ object DirTree: extension [X](thizDt: DirTree[X]) def ->(name: Path.Segment): ZLink[X] = ZLink(thizDt, name) - /** find the closest node X available when following Path return the remaining path - */ + /** find the closest node X available when following Path return the remaining path */ @tailrec def find(at: Path): (Path, X) = at match case Seq() => (at, thizDt.head) @@ -126,8 +125,7 @@ object DirTree: case (Right(_), zpath) => newDt.rezip(zpath) case _ => thizDt - /** set dirTree at path creating new directories with default values if needed - */ + /** set dirTree at path creating new directories with default values if needed */ def insertDirAt(path: Path, dt: DirTree[X], default: X): DirTree[X] = thizDt.unzipAlong(path) match case (Left(path), zpath) => dt.rezip(path.map(p => pure(default) -> p).appendedAll(zpath)) diff --git a/n3js/src/main/scala/n3js/std/package.scala b/n3js/src/main/scala/n3js/std/package.scala index 47dc5ef..2d4968b 100644 --- a/n3js/src/main/scala/n3js/std/package.scala +++ b/n3js/src/main/scala/n3js/std/package.scala @@ -7,14 +7,12 @@ import scala.scalajs.js.annotation.{JSGlobalScope, JSGlobal, JSImport, JSName, J package object std: - /** Convert string literal type to lowercase - */ + /** Convert string literal type to lowercase */ type Lowercase[ S /* <: java.lang.String */ ] = /* import warning: transforms.QualifyReferences#resolveTypeRef many Couldn't qualify intrinsic */ js.Any - /** Construct a type with a set of properties K of type T - */ + /** Construct a type with a set of properties K of type T */ type Record[K /* <: /* keyof any */ java.lang.String */, T] = org.scalablytyped.runtime.StringDictionary[T] From 53c9e5f040c2416e76e72246d5df80d79956580c Mon Sep 17 00:00:00 2001 From: Henry Story Date: Mon, 5 Jun 2023 09:59:14 +0200 Subject: [PATCH 41/42] update to scalafmt "3.7.4" --- .scalafmt.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.scalafmt.conf b/.scalafmt.conf index b53736a..3da5b83 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,4 +1,4 @@ -version = "3.7.3" +version = "3.7.4" runner.dialect = scala3 indent { main = 2 @@ -15,7 +15,7 @@ rewrite.scala3 { convertToNewSyntax = true removeOptionalBraces = yes } -runner.dialectOverride.allowQuestionMarkAsTypeWildcard = false +# runner.dialectOverride.allowQuestionMarkAsTypeWildcard = false newlines { selectChains = fold beforeMultiline = fold From cb2643e9ba9b63c0c1839daf005db9af0dca250c Mon Sep 17 00:00:00 2001 From: Henry Story Date: Wed, 7 Jun 2023 11:15:37 +0200 Subject: [PATCH 42/42] add loging for cache hits --- .scalafmt.conf | 2 +- .../scala/io/chrisdavenport/mules/Cache.scala | 15 ++++ .../mules/http4s/CacheItem.scala | 3 +- .../mules/http4s/internal/Caching.scala | 2 +- .../cache/InterpretedCacheMiddleware.scala | 4 +- .../run/cosy/http/cache/TreeDirCache.scala | 7 +- .../main/scala/run/cosy/ld/http4s/H4Web.scala | 7 +- .../main/scala/scripts/AnHttpSigClient.scala | 90 +++++++++++++------ .../net/bblfish/wallet/BasicAuthWallet.scala | 25 +++--- 9 files changed, 104 insertions(+), 51 deletions(-) diff --git a/.scalafmt.conf b/.scalafmt.conf index 3da5b83..dde6947 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -15,7 +15,7 @@ rewrite.scala3 { convertToNewSyntax = true removeOptionalBraces = yes } -# runner.dialectOverride.allowQuestionMarkAsTypeWildcard = false +runner.dialectOverride.allowQuestionMarkAsTypeWildcard = false newlines { selectChains = fold beforeMultiline = fold diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/Cache.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/Cache.scala index b5f84c5..96d1f68 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/Cache.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/Cache.scala @@ -63,6 +63,16 @@ object Insert: trait Delete[F[_], K]: def delete(k: K): F[Unit] + +trait DeleteBelow[F[_], K]: + /** Deleting a server without a path, results in loosing all info in the path. todo: consider + * if that is really wise. It doe + * todo: this was added for our TreeDirCache - it could make sense as a way of deleting all information below a path + * but that indicates that the key has a structure, which is not visible in the type K, + * so this is a bit of a hack. + */ + def deleteBelow(k: K): F[Unit] + object Delete: def contramap[F[_], A, B](d: Delete[F, A])(g: B => A): Delete[F, B] = new Delete[F, B]: def delete(k: B) = d.delete(g(k)) @@ -79,6 +89,11 @@ trait LocalSearch[F[_], K, V]: trait Cache[F[_], K, V] extends Lookup[F, K, V] with Insert[F, K, V] with Delete[F, K] +/** A Cache that can delete all resources below a key. + * The assumption that the key has that structure should be encoded in the type. + */ +trait DBCache[F[_],K, V] extends Cache[F, K, V] with DeleteBelow[F, K] + object Cache: def imapValues[F[_]: Functor, K, A, B]( cache: Cache[F, K, A] diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala index 6e48f8b..4296f54 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/CacheItem.scala @@ -32,7 +32,8 @@ final case class CacheItem[T]( created: HttpDate, expires: Option[HttpDate], response: CachedResponse[T] -) +): + def map[T2](f: T => T2): CacheItem[T2] = copy(response = response.map(f)) object CacheItem: diff --git a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala index 58855c9..2cf48d1 100644 --- a/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala +++ b/cache/shared/src/main/scala/io/chrisdavenport/mules/http4s/internal/Caching.scala @@ -30,7 +30,7 @@ import run.cosy.http.cache.TreeDirCache import run.cosy.http.cache.ServerNotFound case class Caching[F[_]: Concurrent: Clock, T]( - cache: TreeDirCache[F, CacheItem[T]], + cache: Cache[F, Uri, CacheItem[T]], interpret: Response[F] => F[CachedResponse[T]], cacheType: CacheType ): diff --git a/cache/shared/src/main/scala/run/cosy/http/cache/InterpretedCacheMiddleware.scala b/cache/shared/src/main/scala/run/cosy/http/cache/InterpretedCacheMiddleware.scala index d72af7c..2cc30be 100644 --- a/cache/shared/src/main/scala/run/cosy/http/cache/InterpretedCacheMiddleware.scala +++ b/cache/shared/src/main/scala/run/cosy/http/cache/InterpretedCacheMiddleware.scala @@ -20,6 +20,7 @@ import cats.data.Kleisli import cats.effect.{Clock, Concurrent, Resource} import io.chrisdavenport.mules.http4s.internal.Caching import io.chrisdavenport.mules.http4s.{CacheItem, CacheType, CachedResponse} +import io.chrisdavenport.mules.{Cache, DeleteBelow} import org.http4s.* import org.http4s.client.Client import cats.arrow.FunctionK @@ -30,9 +31,10 @@ import run.cosy.http.cache.TreeDirCache */ object InterpretedCacheMiddleware: type InterpClient[F[_], G[_], T] = Kleisli[G, Request[F], CachedResponse[T]] + type TDCache[F[_], K, V] = Cache[F, K, V] with DeleteBelow[F, K] def client[F[_]: Concurrent: Clock, T]( - cache: TreeDirCache[F, CacheItem[T]], + cache: TDCache[F, Uri, CacheItem[T]], interpret: Response[F] => F[CachedResponse[T]], enhance: Request[F] => Request[F] = identity, cacheType: CacheType = CacheType.Private diff --git a/cache/shared/src/main/scala/run/cosy/http/cache/TreeDirCache.scala b/cache/shared/src/main/scala/run/cosy/http/cache/TreeDirCache.scala index d37725e..edf6ce7 100644 --- a/cache/shared/src/main/scala/run/cosy/http/cache/TreeDirCache.scala +++ b/cache/shared/src/main/scala/run/cosy/http/cache/TreeDirCache.scala @@ -19,7 +19,7 @@ package run.cosy.http.cache import cats.effect.kernel.{Ref, Sync} import cats.syntax.all.* import cats.{FlatMap, MonadError} -import io.chrisdavenport.mules.{Cache, LocalSearch} +import io.chrisdavenport.mules.{Cache, LocalSearch, DeleteBelow, DBCache} import org.http4s.Uri import run.cosy.http.cache.DirTree.* import run.cosy.http.cache.TreeDirCache.WebCache @@ -32,11 +32,10 @@ sealed trait TreeDirException extends Exception case class ServerNotFound(uri: Uri) extends TreeDirException case class IncompleteServiceInfo(uri: Uri) extends TreeDirException -/** this is a mules cache, but we add a method to search the path */ +/** this is a mules cache based on a DirTree */ case class TreeDirCache[F[_], X]( cacheRef: Ref[F, WebCache[X]] -)(using F: Sync[F]) - extends Cache[F, Uri, X]: +)(using F: Sync[F]) extends DBCache[F, Uri, X]: /* todo: consider if that is really wise. It doe */ override def delete(k: Uri): F[Unit] = diff --git a/ldes/shared/src/main/scala/run/cosy/ld/http4s/H4Web.scala b/ldes/shared/src/main/scala/run/cosy/ld/http4s/H4Web.scala index b915225..38f399e 100644 --- a/ldes/shared/src/main/scala/run/cosy/ld/http4s/H4Web.scala +++ b/ldes/shared/src/main/scala/run/cosy/ld/http4s/H4Web.scala @@ -26,9 +26,9 @@ import run.cosy.ld.http4s.RDFDecoders import run.cosy.ld.{UriNGraph, Web} import run.cosy.web.util.UrlUtil.http4sUrlToLLUrl -/** Web implementation in http4s todo: we need a client that can find out about redirects, so that - * we can correctly name the resource todo: we also need a web cache that can be queried and - * updated (or perhaps that would use this) +/** Web implementation in http4s + * + * - todo: we need a client that can find out about redirects, so that we can correctly name the resource */ class H4Web[F[_]: Concurrent, R <: RDF]( client: Client[F] @@ -50,6 +50,7 @@ class H4Web[F[_]: Concurrent, R <: RDF]( yield rG.resolveAgainst(http4sUrlToLLUrl(doc).toAbsoluteUrl) // todo: this should really use client.fetchPNG + // todo: notice this is actually close to the CacheResponse... It would be nice to override def getPNG(url: RDF.URI[R]): F[UriNGraph[R]] = val doc = url.fragmentLess get(doc).map(g => UriNGraph(url, doc, g)) diff --git a/scripts/shared/src/main/scala/scripts/AnHttpSigClient.scala b/scripts/shared/src/main/scala/scripts/AnHttpSigClient.scala index d4a67f4..af8d3ea 100644 --- a/scripts/shared/src/main/scala/scripts/AnHttpSigClient.scala +++ b/scripts/shared/src/main/scala/scripts/AnHttpSigClient.scala @@ -17,32 +17,40 @@ package scripts import _root_.io.lemonlabs.uri as ll +import bobcats.AsymmetricKeyAlg +import bobcats.PKCS8KeySpec +import bobcats.Signer +import bobcats.Verifier import bobcats.util.BouncyJavaPEMUtils.getPrivateKeySpec -import bobcats.{AsymmetricKeyAlg, PKCS8KeySpec, Signer, Verifier} import cats.effect.* import cats.effect.unsafe.IORuntime +import fs2.io.net.Network +import io.chrisdavenport.mules.http4s.CacheItem +import io.chrisdavenport.mules.{Cache, DeleteBelow, DBCache} +import io.chrisdavenport.mules.http4s.CachedResponse import net.bblfish.app.auth.AuthNClient -import net.bblfish.wallet.{BasicId, BasicWallet, KeyData, WalletTools} -import org.w3.banana.jena.JenaRdf -import org.w3.banana.jena.JenaRdf.ops -import ops.given +import net.bblfish.wallet.BasicId +import net.bblfish.wallet.BasicWallet +import net.bblfish.wallet.KeyData +import net.bblfish.wallet.WalletTools +import org.http4s.Response +import org.http4s.client.* +import org.http4s.ember.client.* import org.http4s.Uri as H4Uri +import org.w3.banana.Ops +import org.w3.banana.RDF +import org.w3.banana.jena.JenaRdf import org.w3.banana.jena.JenaRdf.R +import org.w3.banana.jena.JenaRdf.ops +import run.cosy.http.cache.InterpretedCacheMiddleware +import run.cosy.http.cache.TreeDirCache +import run.cosy.http.cache.TreeDirCache.WebCache import run.cosy.http.headers.Rfc8941 +import run.cosy.http.headers.SigIn.KeyId import run.cosy.ld.http4s.RDFDecoders import scodec.bits.ByteVector -import org.http4s.client.* -import org.http4s.ember.client.* -import run.cosy.http.headers.SigIn.KeyId -import org.w3.banana.RDF -import org.w3.banana.Ops -import run.cosy.http.cache.TreeDirCache.WebCache -import io.chrisdavenport.mules.http4s.CacheItem -import run.cosy.http.cache.TreeDirCache -import run.cosy.http.cache.InterpretedCacheMiddleware -import org.http4s.Response -import io.chrisdavenport.mules.http4s.CachedResponse -import fs2.io.net.Network + +import ops.given object AnHttpSigClient: implicit val runtime: IORuntime = cats.effect.unsafe.IORuntime.global @@ -94,13 +102,13 @@ object AnHttpSigClient: given rdfDecoders: RDFDecoders[IO, R] = RDFDecoders[IO, R] def ioStr(uri: H4Uri, client: Client[IO]): IO[String] = ClientTools - .authClient[IO, R](keyIdData, client).flatMap(_.expect[String](uri)) + .authClient[IO, R](keyIdData, client, None).flatMap(_.expect[String](uri)) // run "use" on this to get a client def emberClient: Resource[IO, Client[IO]] = EmberClientBuilder.default[IO].build def emberAuthClient: IO[Client[IO]] = emberClient.use { client => - ClientTools.authClient(keyIdData, client) + ClientTools.authClient(keyIdData, client, Some(str => IO("--log: "+println(str)))) } @main @@ -121,27 +129,51 @@ object ClientTools: /** enhance client so that it logs and can authenticate with given keyId, and save acls to graph * cache it creates. (a bit adhoc of a function, but it is a script) + * todo: generalise this to a client that can take a set of keyIds and passwords */ def authClient[F[_]: Clock: Async, R <: RDF]( keyIdData: KeyData[F], - client: Client[F] + client: Client[F], + log: Option[String => F[Unit]] )(using ops: Ops[R], rdfDecoders: RDFDecoders[F, R] ): F[Client[F]] = import org.http4s.client.middleware.Logger - val loggedClient: Client[F] = Logger[F]( - true, - true, - logAction = Some(str => Concurrent[F].pure(System.out.println(str))) - )(client) + // val loggedClient: Client[F] = Logger[F]( + // true, + // true, + // logAction = Some(str => Concurrent[F].pure(System.out.println(str))) + // )(client) val walletTools: WalletTools[R] = new WalletTools[R] for ref <- Ref.of[F, WebCache[CacheItem[RDF.rGraph[R]]]](Map.empty) yield - val cache = TreeDirCache[F, CacheItem[RDF.rGraph[R]]](ref) - val interClientMiddleware = walletTools.cachedRelGraphMiddleware(cache) + val cache: DBCache[F, H4Uri, CacheItem[RDF.rGraph[R]]] = + TreeDirCache[F, CacheItem[RDF.rGraph[R]]](ref) + val lcache: DBCache[F, H4Uri, CacheItem[RDF.rGraph[R]]] = log match + case None => cache + case Some(f) => new DBCache[F, H4Uri, CacheItem[RDF.rGraph[R]]]: + // import flatTap syntax here + import cats.syntax.all.* + override def lookup(k: H4Uri): F[Option[CacheItem[RDF.rGraph[R]]]] = cache.lookup(k) + .flatTap { opt => + opt match + case Some(item) => + f(s"Cache hit for <$k> with ${item.map(_ => "--not showing triples--")}") + case None => f(s"Cache miss for <$k>") + } + override def insert( + k: H4Uri, + v: CacheItem[RDF.rGraph[R]] + ): F[Unit] = cache.insert(k, v).flatTap(_ => + f(s"Cache insert for <$k> value ${v.map(_ => "--not showing triples")} ") + ) + override def delete(k: H4Uri): F[Unit] = cache.delete(k) + override def deleteBelow(k: H4Uri): F[Unit] = cache.deleteBelow(k) + + val interClientMiddleware = walletTools.cachedRelGraphMiddleware(lcache) val bw = new BasicWallet[F, R]( Map(), Seq(keyIdData) - )(interClientMiddleware(loggedClient)) - AuthNClient[F](bw)(loggedClient) + )(interClientMiddleware(client)) + AuthNClient[F](bw)(client) diff --git a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala index 1f3856e..68b0b2f 100644 --- a/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala +++ b/wallet/shared/src/main/scala/net/bblfish/wallet/BasicAuthWallet.scala @@ -50,12 +50,13 @@ import scala.concurrent.duration.FiniteDuration import scala.reflect.TypeTest import scala.util.{Failure, Try} import io.chrisdavenport.mules.http4s.CacheItem +import io.chrisdavenport.mules.http4s.CachedResponse +import io.chrisdavenport.mules.{Cache, DeleteBelow} import run.cosy.http.cache.TreeDirCache import run.cosy.http.cache.InterpretedCacheMiddleware.InterpClient import org.http4s.HttpApp import cats.effect.kernel.Resource import run.cosy.http.cache.InterpretedCacheMiddleware -import io.chrisdavenport.mules.http4s.CachedResponse import cats.MonadThrow import org.http4s.EntityDecoder import net.bblfish.wallet.BasicWallet.defaultAC @@ -104,6 +105,7 @@ object BasicWallet: val defaultAC = "defaultAccessContainer" val defaultACOpt = Some(defaultAC) val ldpContains = "http://www.w3.org/ns/ldp#contains" + type TDCache[F[_], K, V] = Cache[F, K, V] with DeleteBelow[F, K] /** The type of relations that can be found in the Link header, Left is reverse and Right is * forward, and the link is to the absoluteUrl from the document @@ -162,7 +164,7 @@ class WalletTools[Rdf <: RDF](using ops: Ops[Rdf]): * Todo: also store graphs for other responses? (e.g. error responses?) */ def cachedRelGraphMiddleware[F[_]: Concurrent: Clock]( - cache: TreeDirCache[F, CacheItem[RDF.rGraph[Rdf]]] + cache: TDCache[F, Uri, CacheItem[RDF.rGraph[Rdf]]] )(using rdfDecoders: RDFDecoders[F, Rdf] ): Client[F] => InterpClient[F, Resource[F, *], RDF.rGraph[Rdf]] = InterpretedCacheMiddleware @@ -536,14 +538,15 @@ class BasicWallet[F[_], Rdf <: RDF]( end sign override def signFromDB(req: h4s.Request[F]): F[Either[Throwable, h4s.Request[F]]] = - // todo: the DB needs to keep track of what WWW-Authenticate methods the server allows. - // These will be difficult to find in the headers, as the 401 in which they appeared may be - // somewhere completely different. - // I will assume that the server can do HTTP-Sig for the moment - // todo: fix!!! - // todo: as we endup with F[Request] do we need Resources everywhere here? - findCachedAclFor(req.uri.toLL.toAbsoluteUrl, 100).flatMap { - signRequest(req, req.uri.toLL.toAbsoluteUrl) - }.attempt.use(fc.pure) + // todo: the DB needs to keep track of what WWW-Authenticate methods the server allows. + // These will be difficult to find in the headers, as the 401 in which they appeared may be + // somewhere completely different. + // I will assume that the server can do HTTP-Sig for the moment + // todo: fix!!! + // todo: as we endup with F[Request] do we need Resources everywhere here? + val absoluteReqUrl = req.uri.toLL.toAbsoluteUrl + findCachedAclFor(absoluteReqUrl, 100).flatMap { + signRequest(req, absoluteReqUrl) + }.attempt.use(fc.pure) end BasicWallet