From af9dee164e16fbe79ec14c7ec39f291894575357 Mon Sep 17 00:00:00 2001 From: Brian Schlining Date: Tue, 15 Oct 2024 17:47:58 -0700 Subject: [PATCH] check point --- annosaurus/src/main/resources/reference.conf | 3 + .../org/mbari/annosaurus/AppConfig.scala | 4 +- .../org/mbari/annosaurus/Endpoints.scala | 8 +- .../scala/org/mbari/annosaurus/Main.scala | 3 +- .../controllers/AnnotationController.scala | 3 +- .../controllers/QueryController.scala | 42 ++++++ .../domain/ObservationsUpdate.scala | 20 ++- .../annosaurus/domain/QueryRequest.scala | 24 +++ .../annosaurus/endpoints/Endpoints.scala | 23 +++ .../endpoints/FastAnnotationEndpoints.scala | 8 +- .../endpoints/ImagedMomentEndpoints.scala | 4 +- .../endpoints/ObservationEndpoints.scala | 18 ++- .../annosaurus/endpoints/QueryEndpoints.scala | 78 ++++++++++ .../annosaurus/etc/circe/CirceCodecs.scala | 62 ++++++-- .../annosaurus/etc/jpa/EntityManagers.scala | 60 ++++++++ .../repository/jdbc/AnalysisRepository.scala | 16 +- .../repository/jdbc/JdbcRepository.scala | 46 +++--- .../repository/jdbc/ObservationSQL.scala | 13 +- .../repository/jdbc/TimeHistogramSQL.scala | 6 +- .../jpa/CachedVideoReferenceInfoDAOImpl.scala | 3 +- .../jpa/EntityManagerFactories.scala | 16 +- .../repository/jpa/ObservationDAOImpl.scala | 9 +- .../annosaurus/repository/query/JDBC.scala | 141 +++++++++++------- .../query/PreparedStatementGenerator.scala | 49 ++++-- .../repository/query/QueryResults.scala | 16 +- .../repository/query/QueryService.scala | 100 ++++++++----- .../repository/query/constraints.scala | 43 ++++-- .../src/test/resources/json/constraints.json | 17 +++ .../etc/circe/CirceCodecsSuite.scala | 3 +- build.sbt | 2 +- .../AnnotationControllerSuite.scala | 4 +- .../endpoints/AnalysisEndpointsSuite.scala | 26 ++-- .../endpoints/AnnotationEndpointsSuite.scala | 23 ++- .../ImagedMomentEndpointsSuite.scala | 4 +- .../endpoints/ObservationEndpointsSuite.scala | 34 +++-- .../jdbc/AnalysisRepositorySuite.scala | 15 +- .../repository/jdbc/JdbcRepositorySuite.scala | 23 +-- project/Dependencies.scala | 4 +- 38 files changed, 722 insertions(+), 251 deletions(-) create mode 100644 annosaurus/src/main/scala/org/mbari/annosaurus/controllers/QueryController.scala create mode 100644 annosaurus/src/main/scala/org/mbari/annosaurus/domain/QueryRequest.scala create mode 100644 annosaurus/src/main/scala/org/mbari/annosaurus/endpoints/QueryEndpoints.scala create mode 100644 annosaurus/src/main/scala/org/mbari/annosaurus/etc/jpa/EntityManagers.scala diff --git a/annosaurus/src/main/resources/reference.conf b/annosaurus/src/main/resources/reference.conf index 0c10c877..99f14190 100644 --- a/annosaurus/src/main/resources/reference.conf +++ b/annosaurus/src/main/resources/reference.conf @@ -48,6 +48,9 @@ database { url = ${?DATABASE_URL} user = "sa" user = ${?DATABASE_USER} + + query.view = ${?DATABASE_QUERY_VIEW} + query.view = "annotations" # name = "Derby" # name = ${?DATABASE_NAME} # https://docs.jboss.org/hibernate/orm/4.3/manual/en-US/html_single/#configuration-optional-dialects diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/AppConfig.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/AppConfig.scala index dfe3194c..3bd71ac2 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/AppConfig.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/AppConfig.scala @@ -73,6 +73,7 @@ object AppConfig { user = Config.getString("database.user"), password = Config.getString("database.password"), driver = Config.getString("database.driver"), + queryView = Config.getString("database.query.view") ) } @@ -87,7 +88,8 @@ case class DatabaseConfig( url: String, user: String, password: String, - driver: String + driver: String, + queryView: String ): def newConnection(): java.sql.Connection = Class.forName(driver) diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/Endpoints.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/Endpoints.scala index f02c8ed8..e04810b8 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/Endpoints.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/Endpoints.scala @@ -44,6 +44,10 @@ object Endpoints { val imageReferenceController = new ImageReferenceController(daoFactory) val indexController = new IndexController(daoFactory) val observationController = new ObservationController(daoFactory) + val queryController = new QueryController( + AppConfig.DefaultDatabaseConfig, + AppConfig.DefaultDatabaseConfig.queryView + ) // -------------------------------- val analysisRepository = new AnalysisRepository(daoFactory.entityManagerFactory) @@ -67,6 +71,7 @@ object Endpoints { val imageReferenceEndpoints = new ImageReferenceEndpoints(imageReferenceController) val indexEndpoints = new IndexEndpoints(indexController) val observationEndpoints = new ObservationEndpoints(observationController, jdbcRepository) + val queryEndpoints = new QueryEndpoints(queryController) // -------------------------------- // For VertX, we need to separate the non-blocking endpoints from the blocking ones @@ -86,7 +91,8 @@ object Endpoints { imageEndpoints.allImpl, imageReferenceEndpoints.allImpl, indexEndpoints.allImpl, - observationEndpoints.allImpl + observationEndpoints.allImpl, + queryEndpoints.allImpl ).flatten val apiEndpoints = nonBlockingEndpoints ++ blockingEndpoints diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/Main.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/Main.scala index 98d34264..6c3b0379 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/Main.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/Main.scala @@ -63,7 +63,8 @@ object Main: val port = sys.env.get("HTTP_PORT").flatMap(_.toIntOption).getOrElse(8080) log.atInfo.log(s"Starting ${AppConfig.Name} v${AppConfig.Version} on port $port") - val vertx = Vertx.vertx(new VertxOptions().setWorkerPoolSize(AppConfig.NumberOfVertxWorkers)) + val vertx = + Vertx.vertx(new VertxOptions().setWorkerPoolSize(AppConfig.NumberOfVertxWorkers)) // val vertx = Vertx.vertx() val server = vertx.createHttpServer() val router = Router.router(vertx) diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/controllers/AnnotationController.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/controllers/AnnotationController.scala index f1d14ed5..ca98fe24 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/controllers/AnnotationController.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/controllers/AnnotationController.scala @@ -249,7 +249,7 @@ class AnnotationController( def create(annotation: Annotation)(using ec: ExecutionContext): Future[Seq[Annotation]] = val entity = annotation.toEntity - val dao = daoFactory.newImagedMomentDAO() + val dao = daoFactory.newImagedMomentDAO() val future = dao.runTransaction(d => { // d.create(entity) // Annotation.fromImagedMoment(entity, true) @@ -259,7 +259,6 @@ class AnnotationController( future.onComplete(_ => dao.close()) future - /** Bulk create annotations * @param annotations * THe annotations to create diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/controllers/QueryController.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/controllers/QueryController.scala new file mode 100644 index 00000000..7e499c96 --- /dev/null +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/controllers/QueryController.scala @@ -0,0 +1,42 @@ +/* + * Copyright 2017 Monterey Bay Aquarium Research Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.mbari.annosaurus.controllers + +import org.mbari.annosaurus.DatabaseConfig +import org.mbari.annosaurus.domain.QueryRequest +import org.mbari.annosaurus.repository.query.{ + Constraint, + Constraints, + JDBC, + QueryResults, + QueryService +} + +class QueryController(databaseConfig: DatabaseConfig, viewName: String) { + + private lazy val queryService = new QueryService(databaseConfig, viewName) + + def count(constraints: Constraints): Either[Throwable, Int] = + queryService.count(constraints) + + def query(queryRequest: QueryRequest): Either[Throwable, QueryResults] = + queryService.query(queryRequest.querySelects, queryRequest.constraints) + + def listColumns(): Either[Throwable, Seq[JDBC.Metadata]] = + queryService.jdbc.listColumnsMetadata(viewName) + +} diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/domain/ObservationsUpdate.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/domain/ObservationsUpdate.scala index d3ae2004..32b2c019 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/domain/ObservationsUpdate.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/domain/ObservationsUpdate.scala @@ -18,14 +18,18 @@ package org.mbari.annosaurus.domain import java.util.UUID -/** - * Update multiple observations at once - * @param observationUuids The UUIDs of the observations to update - * @param concept The new concept - * @param observer The new observer - * @param activity The new activity - * @param group The new group - */ +/** Update multiple observations at once + * @param observationUuids + * The UUIDs of the observations to update + * @param concept + * The new concept + * @param observer + * The new observer + * @param activity + * The new activity + * @param group + * The new group + */ case class ObservationsUpdate( observationUuids: Seq[UUID], concept: Option[String] = None, diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/domain/QueryRequest.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/domain/QueryRequest.scala new file mode 100644 index 00000000..32dd9cb3 --- /dev/null +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/domain/QueryRequest.scala @@ -0,0 +1,24 @@ +/* + * Copyright 2017 Monterey Bay Aquarium Research Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.mbari.annosaurus.domain + +import org.mbari.annosaurus.repository.query.Constraints + +case class QueryRequest( + querySelects: Seq[String], + constraints: Constraints +) diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/endpoints/Endpoints.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/endpoints/Endpoints.scala index 61694ad8..523b4cf6 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/endpoints/Endpoints.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/endpoints/Endpoints.scala @@ -24,6 +24,7 @@ import org.mbari.annosaurus.etc.circe.CirceCodecs import org.mbari.annosaurus.etc.circe.CirceCodecs.{*, given} import org.mbari.annosaurus.etc.jwt.JwtService import org.mbari.annosaurus.etc.jdk.Logging.given +import org.mbari.annosaurus.repository.query.{Constraint, Constraints} import scala.concurrent.ExecutionContext import scala.concurrent.Future @@ -118,6 +119,19 @@ trait Endpoints: implicit lazy val sURL: Schema[URL] = Schema.string implicit lazy val sInstant: Schema[Instant] = Schema.string implicit lazy val sBulkAnnotationSc: Schema[BulkAnnotationSC] = Schema.derived[BulkAnnotationSC] + implicit lazy val sConstraintDate: Schema[Constraint.Date] = Schema.derived[Constraint.Date] + implicit lazy val sConstraintInString: Schema[Constraint.In[String]] = + Schema.derived[Constraint.In[String]] + implicit lazy val sConstraintLike: Schema[Constraint.Like] = Schema.derived[Constraint.Like] + implicit lazy val sConstraintMax: Schema[Constraint.Max] = Schema.derived[Constraint.Max] + implicit lazy val sConstraintMin: Schema[Constraint.Min] = Schema.derived[Constraint.Min] + implicit lazy val sConstraintMinMax: Schema[Constraint.MinMax] = + Schema.derived[Constraint.MinMax] + implicit lazy val sConstraintIsNull: Schema[Constraint.IsNull] = + Schema.derived[Constraint.IsNull] + implicit lazy val sConstraint: Schema[Constraint] = Schema.string + implicit lazy val sConstraints: Schema[Constraints] = Schema.derived[Constraints] + implicit lazy val sQueryRequest: Schema[QueryRequest] = Schema.derived[QueryRequest] // given Schema[Option[URL]] = Schema.string // implicit lazy val sOptCAD: Schema[Option[CachedAncillaryDatumSC]] = Schema.derived[Option[CachedAncillaryDatumSC]] // implicit lazy val sOptDouble: Schema[Option[Double]] = Schema.derived[Option[Double]] @@ -139,6 +153,15 @@ trait Endpoints: log.atError.withCause(exception).log("Error") Success(Left(ServerError(exception.getMessage))) + def handleEitherAsync[T]( + f: => Either[Throwable, T] + )(using ec: ExecutionContext): Future[Either[ErrorMsg, T]] = + Future { + f match + case Right(value) => Right(value) + case Left(e) => Left(ServerError(e.getMessage)) + } + def handleOption[T](f: Future[Option[T]])(using ec: ExecutionContext ): Future[Either[ErrorMsg, T]] = diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/endpoints/FastAnnotationEndpoints.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/endpoints/FastAnnotationEndpoints.scala index 25f1e050..13ebc90a 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/endpoints/FastAnnotationEndpoints.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/endpoints/FastAnnotationEndpoints.scala @@ -335,7 +335,9 @@ class FastAnnotationEndpoints(jdbcRepository: JdbcRepository)(using .in(paging) .out(jsonBody[Seq[UUID]]) .name("findImageMomentUuidsByConcept") - .description("Find the UUIDS of image moments by concept. Only include image moments with images. Sorted by recorded timestamp.") + .description( + "Find the UUIDS of image moments by concept. Only include image moments with images. Sorted by recorded timestamp." + ) .tag(tag) val findImagedMomentUuidsByConceptImpl: ServerEndpoint[Any, Future] = @@ -363,7 +365,9 @@ class FastAnnotationEndpoints(jdbcRepository: JdbcRepository)(using .in(paging) .out(jsonBody[Seq[UUID]]) .name("findImagedMomentUuidsByToConcept") - .description("Find image moment UUIDs by to concept. Only include image moments with images. Sorted by recorded timestamp.") + .description( + "Find image moment UUIDs by to concept. Only include image moments with images. Sorted by recorded timestamp." + ) .tag(tag) val findImagedMomentUuidsByToConceptImpl: ServerEndpoint[Any, Future] = diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/endpoints/ImagedMomentEndpoints.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/endpoints/ImagedMomentEndpoints.scala index 842ee062..f1f3573f 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/endpoints/ImagedMomentEndpoints.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/endpoints/ImagedMomentEndpoints.scala @@ -59,7 +59,9 @@ class ImagedMomentEndpoints(controller: ImagedMomentController)(using .in(jsonBody[MoveImagedMoments]) .out(jsonBody[Count]) .name("bulkMove") - .description("Bulk move imaged moments to a new video reference. JSON request can be camelCase or snake_case") + .description( + "Bulk move imaged moments to a new video reference. JSON request can be camelCase or snake_case" + ) .tag(tag) val bulkMoveImpl: ServerEndpoint[Any, Future] = diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/endpoints/ObservationEndpoints.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/endpoints/ObservationEndpoints.scala index daa89476..06e23638 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/endpoints/ObservationEndpoints.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/endpoints/ObservationEndpoints.scala @@ -17,7 +17,17 @@ package org.mbari.annosaurus.endpoints import org.mbari.annosaurus.controllers.ObservationController -import org.mbari.annosaurus.domain.{ConceptCount, Count, CountForVideoReferenceSC, ErrorMsg, ObservationSC, ObservationUpdateSC, ObservationsUpdate, RenameConcept, RenameCountSC} +import org.mbari.annosaurus.domain.{ + ConceptCount, + Count, + CountForVideoReferenceSC, + ErrorMsg, + ObservationSC, + ObservationUpdateSC, + ObservationsUpdate, + RenameConcept, + RenameCountSC +} import org.mbari.annosaurus.etc.jwt.JwtService import org.mbari.annosaurus.etc.tapir.TapirCodecs.given import sttp.tapir.* @@ -320,7 +330,11 @@ class ObservationEndpoints(controller: ObservationController, jdbcRepository: Jd secureEndpoint .put .in(base / "bulk") - .in(jsonBody[ObservationsUpdate].description("Describes the parameters and uuids of the observations to update. Can be camelCase or snake_case.")) + .in( + jsonBody[ObservationsUpdate].description( + "Describes the parameters and uuids of the observations to update. Can be camelCase or snake_case." + ) + ) .out(jsonBody[Count].description("The number of observations updated")) .name("updateManyObservations") .description( diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/endpoints/QueryEndpoints.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/endpoints/QueryEndpoints.scala new file mode 100644 index 00000000..3433c2d7 --- /dev/null +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/endpoints/QueryEndpoints.scala @@ -0,0 +1,78 @@ +/* + * Copyright 2017 Monterey Bay Aquarium Research Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.mbari.annosaurus.endpoints + +import CustomTapirJsonCirce.* +import org.mbari.annosaurus.controllers.QueryController +import org.mbari.annosaurus.domain.{ErrorMsg, QueryRequest} +import org.mbari.annosaurus.etc.circe.CirceCodecs.{*, given} +import org.mbari.annosaurus.etc.tapir.TapirCodecs.given +import org.mbari.annosaurus.repository.query.{JDBC, QueryResults} +import sttp.model.StatusCode +import sttp.tapir.* +import sttp.tapir.generic.auto.* +import sttp.tapir.server.ServerEndpoint +import sttp.tapir.server.ServerEndpoint.Full + +import scala.util.Success +import scala.concurrent.{ExecutionContext, Future} + +class QueryEndpoints(queryController: QueryController)(using executionContext: ExecutionContext) + extends Endpoints { + + private val base = "query" + private val tag = "Query" + + val listColumns: Endpoint[Unit, Unit, ErrorMsg, Seq[JDBC.Metadata], Any] = openEndpoint + .get + .in(base / "columns") + .out(jsonBody[Seq[JDBC.Metadata]]) + .name("listQueryColumns") + .description("List columns in the query view") + .tag(tag) + + val listColumnsImpl: Full[Unit, Unit, Unit, ErrorMsg, Seq[JDBC.Metadata], Any, Future] = + listColumns.serverLogic(_ => handleEitherAsync(queryController.listColumns())) + + val runQuery: Endpoint[Unit, QueryRequest, ErrorMsg, String, Any] = + openEndpoint + .post + .in(base / "run") + .in(jsonBody[QueryRequest]) + .out(stringBody) + .out(header("Content-Type", "text/tab-separated-values")) + .name("runQuery") + .description("Run a query") + .tag(tag) + + val runQueryImpl = + runQuery.serverLogic(request => + handleEitherAsync( + queryController.query(request).map(QueryResults.toTsv).map(_.mkString("\n")) + ) + ) + + override def all: List[Endpoint[_, _, _, _, _]] = List( + listColumns, + runQuery + ) + + override def allImpl: List[ServerEndpoint[Any, Future]] = List( + listColumnsImpl, + runQueryImpl + ) +} diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/etc/circe/CirceCodecs.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/etc/circe/CirceCodecs.scala index 2fcadb6e..aa40ace7 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/etc/circe/CirceCodecs.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/etc/circe/CirceCodecs.scala @@ -21,7 +21,7 @@ import io.circe.generic.semiauto.* import io.circe.syntax.* import org.mbari.annosaurus.util.HexUtil import org.mbari.annosaurus.domain.* -import org.mbari.annosaurus.repository.query.{Constraint, Constraints} +import org.mbari.annosaurus.repository.query.{Constraint, Constraints, JDBC} import java.net.{URI, URL} import java.time.Instant @@ -339,30 +339,70 @@ object CirceCodecs { constraint <- { if (c.downField("in").succeeded) { c.downField("in").as[List[String]].map(Constraint.In(columnName, _)) - } else if (c.downField("like").succeeded) { + } + else if (c.downField("like").succeeded) { c.downField("like").as[String].map(Constraint.Like(columnName, _)) - } else if (c.downField("between").succeeded) { + } + else if (c.downField("between").succeeded) { // Attempt to decode between as List[Int] first - c.downField("between").as[List[Double]].map(xs => Constraint.MinMax(columnName, xs.head, xs.last)) + c.downField("between") + .as[List[Double]] + .map(xs => Constraint.MinMax(columnName, xs.head, xs.last)) .orElse { // If decoding as Int fails, try decoding as List[Instant] - c.downField("between").as[List[Instant]].map(xs => Constraint.Date(columnName, xs.head, xs.last)) + c.downField("between") + .as[List[Instant]] + .map(xs => Constraint.Date(columnName, xs.head, xs.last)) } - } else if (c.downField("minmax").succeeded) { - c.downField("minmax").as[List[Double]].map(xs => Constraint.MinMax(columnName, xs.head, xs.last)) - } else if (c.downField("min").succeeded) { + } + else if (c.downField("minmax").succeeded) { + c.downField("minmax") + .as[List[Double]] + .map(xs => Constraint.MinMax(columnName, xs.head, xs.last)) + } + else if (c.downField("min").succeeded) { c.downField("min").as[Double].map(Constraint.Min(columnName, _)) - } else if (c.downField("max").succeeded) { + } + else if (c.downField("max").succeeded) { c.downField("max").as[Double].map(Constraint.Max(columnName, _)) - } else { + } + else if (c.downField("isnull").succeeded) { + c.downField("isnull").as[Boolean].map(Constraint.IsNull(columnName, _)) + } + else { Left(DecodingFailure("Unknown constraint type", c.history)) } } } yield constraint } - + + given Encoder[Constraint.Date] = deriveEncoder + given Encoder[Constraint.In[String]] = deriveEncoder + given Encoder[Constraint.Like] = deriveEncoder + given Encoder[Constraint.Max] = deriveEncoder + given Encoder[Constraint.Min] = deriveEncoder + given Encoder[Constraint.MinMax] = deriveEncoder + given Encoder[Constraint.IsNull] = deriveEncoder + given Encoder[List[Constraint]] = deriveEncoder + + // THis is needed to handle the trait Constraint used in Constraints + given constraintEncoder: Encoder[Constraint] = Encoder.instance[Constraint] { + case c: Constraint.In[String] => c.asJson + case c: Constraint.Like => c.asJson + case c: Constraint.Min => c.asJson + case c: Constraint.Max => c.asJson + case c: Constraint.MinMax => c.asJson + case c: Constraint.IsNull => c.asJson + case c: Constraint.Date => c.asJson + } + given constraintsDecoder: Decoder[Constraints] = deriveDecoder + given constraintsEncoder: Encoder[Constraints] = deriveEncoder + given queryRequestDecoder: Decoder[QueryRequest] = deriveDecoder + given queryRequestEncoder: Encoder[QueryRequest] = deriveEncoder + given jdbcMetadataDecoder: Decoder[JDBC.Metadata] = deriveDecoder + given jdbcMetadataEncoder: Encoder[JDBC.Metadata] = deriveEncoder val CustomPrinter: Printer = Printer( dropNullValues = true, diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/etc/jpa/EntityManagers.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/etc/jpa/EntityManagers.scala new file mode 100644 index 00000000..2488b9ff --- /dev/null +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/etc/jpa/EntityManagers.scala @@ -0,0 +1,60 @@ +/* + * Copyright 2017 Monterey Bay Aquarium Research Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.mbari.annosaurus.etc.jpa + +import jakarta.persistence.EntityManager +import org.checkerframework.checker.units.qual.N + +import scala.util.control.NonFatal +import org.mbari.annosaurus.etc.jdk.Logging.given +import org.hibernate.Session + +object EntityManagers: + + private val log = System.getLogger(getClass.getName) + + extension (entityManager: EntityManager) + def runTransaction[R](fn: EntityManager => R): Either[Throwable, R] = + + val transaction = entityManager.getTransaction + transaction.begin() + try + val n = fn.apply(entityManager) + transaction.commit() + Right(n) + catch + case NonFatal(e) => + log.atError.withCause(e).log("Error in transaction: " + e.getCause) + Left(e) + finally if transaction.isActive then transaction.rollback() + + // def runQuery[R](fn: EntityManager => R): Either[Throwable, R] = + // entityManager.unwrap(classOf[Session]).setDefaultReadOnly(true) + // val transaction = entityManager.getTransaction + + // try + // transaction.begin() + // val n = fn.apply(entityManager) + // // transaction.commit() + // Right(n) + // catch + // case NonFatal(e) => + // log.atError.withCause(e).log("Error in transaction: " + e.getCause) + // Left(e) + // finally + // if transaction.isActive then + // transaction.rollback() diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jdbc/AnalysisRepository.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jdbc/AnalysisRepository.scala index 934f28ae..954e8ad3 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jdbc/AnalysisRepository.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jdbc/AnalysisRepository.scala @@ -33,33 +33,35 @@ class AnalysisRepository(entityManagerFactory: EntityManagerFactory) { def depthHistogram(constraints: QueryConstraints, binSizeMeters: Int = 50): DepthHistogram = { val select = DepthHistogramSQL.selectFromBinSize(binSizeMeters) val entityManager: EntityManager = entityManagerFactory.createEntityManager() - val transaction = entityManager.getTransaction() + val transaction = entityManager.getTransaction() transaction.begin() val query = QueryConstraintsSqlBuilder.toQuery(constraints, entityManager, select, "") query.setHint(QueryHints.HINT_READONLY, true) val results = query.getResultList.iterator().next() transaction.commit() entityManager.close() - val values = results.asInstanceOf[Array[Object]] + val values = results + .asInstanceOf[Array[Object]] .map(s => s.asInt.getOrElse(0)) .toList - val binsMin = (0 until DepthHistogramSQL.MaxDepth by binSizeMeters).toList - val binsMax = binsMin.map(_ + binSizeMeters) + val binsMin = (0 until DepthHistogramSQL.MaxDepth by binSizeMeters).toList + val binsMax = binsMin.map(_ + binSizeMeters) DepthHistogram(binsMin, binsMax, values) } def timeHistogram(constraints: QueryConstraints, binSizeDays: Int = 30): TimeHistogram = { - val start = constraints.minTimestamp.getOrElse(TimeHistogramSQL.MinTime) + val start = constraints.minTimestamp.getOrElse(TimeHistogramSQL.MinTime) val now = constraints.maxTimestamp.getOrElse(Instant.now()) val select = TimeHistogramSQL.selectFromBinSize(start, now, binSizeDays) val entityManager: EntityManager = entityManagerFactory.createEntityManager() val query = QueryConstraintsSqlBuilder.toQuery(constraints, entityManager, select, "") query.setHint(QueryHints.HINT_READONLY, true) // println(query) - val results = query.getResultList.iterator().next() + val results = query.getResultList.iterator().next() entityManager.close() - val values = results.asInstanceOf[Array[Object]] + val values = results + .asInstanceOf[Array[Object]] .map(s => s.asInt.getOrElse(0)) .toList diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jdbc/JdbcRepository.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jdbc/JdbcRepository.scala index 864c9821..cb6c00d3 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jdbc/JdbcRepository.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jdbc/JdbcRepository.scala @@ -22,8 +22,17 @@ import jakarta.persistence.{EntityManager, EntityManagerFactory, Query} import scala.jdk.CollectionConverters.* import scala.util.control.NonFatal -import org.mbari.annosaurus.domain.{Annotation, ConcurrentRequest, DeleteCount, GeographicRange, Image, MultiRequest, ObservationsUpdate, QueryConstraints} -import org.mbari.annosaurus.etc.jdk.Logging.{given, *} +import org.mbari.annosaurus.domain.{ + Annotation, + ConcurrentRequest, + DeleteCount, + GeographicRange, + Image, + MultiRequest, + ObservationsUpdate, + QueryConstraints +} +import org.mbari.annosaurus.etc.jdk.Logging.{*, given} import org.mbari.annosaurus.repository.jpa.extensions.* // import org.mbari.annosaurus.etc.jpa.EntityManagers.* import jakarta.persistence.QueryHint @@ -39,12 +48,11 @@ class JdbcRepository(entityManagerFactory: EntityManagerFactory) { private val log = System.getLogger(getClass.getName) - def updateObservations(update: ObservationsUpdate): Int = { implicit val entityManager: EntityManager = entityManagerFactory.createEntityManager() - val n = entityManager.runTransactionSync { em => + val n = entityManager.runTransactionSync { em => val queries = ObservationSQL.buildUpdates(update, entityManager) - val counts = queries.map(_.executeUpdate()) + val counts = queries.map(_.executeUpdate()) counts.headOption.getOrElse(0) } entityManager.close() @@ -103,22 +111,23 @@ class JdbcRepository(entityManagerFactory: EntityManagerFactory) { def findByQueryConstraint(constraints: QueryConstraints): Seq[Annotation] = { given entityManager: EntityManager = entityManagerFactory.createEntityManager() entityManagerFactory.createEntityManager().runTransactionSync { entityManager => - given EntityManager = entityManager - val query1 = QueryConstraintsSqlBuilder.toQuery(constraints, entityManager) - val r1 = query1.getResultList.asScala.toList - val annotations = AnnotationSQL.resultListToAnnotations(r1) - val resolvedAnnotations = executeQueryForAnnotations(annotations, constraints.includeData) + given EntityManager = entityManager + val query1 = QueryConstraintsSqlBuilder.toQuery(constraints, entityManager) + val r1 = query1.getResultList.asScala.toList + val annotations = AnnotationSQL.resultListToAnnotations(r1) + val resolvedAnnotations = + executeQueryForAnnotations(annotations, constraints.includeData) resolvedAnnotations.map(_.removeForeignKeys()) - } + } } def countByQueryConstraint(constraints: QueryConstraints): Int = { entityManagerFactory.createEntityManager().runTransactionSync { entityManager => given EntityManager = entityManager - val query = QueryConstraintsSqlBuilder.toCountQuery(constraints, entityManager) + val query = QueryConstraintsSqlBuilder.toCountQuery(constraints, entityManager) query.setHint(QueryHints.HINT_READONLY, true) // Postgresql returns a Long, Everything else returns an Int - val count = query.getResultList.get(0).toString().toInt + val count = query.getResultList.get(0).toString().toInt entityManager.close() count } @@ -129,10 +138,11 @@ class JdbcRepository(entityManagerFactory: EntityManagerFactory) { ): Option[GeographicRange] = { entityManagerFactory.createEntityManager().runTransactionSync { entityManager => given EntityManager = entityManager - val query = QueryConstraintsSqlBuilder.toGeographicRangeQuery(constraints, entityManager) + val query = + QueryConstraintsSqlBuilder.toGeographicRangeQuery(constraints, entityManager) query.setHint(QueryHints.HINT_READONLY, true) // Queries return java.util.List[Array[Object]] - val count = query.getResultList.asScala.toList + val count = query.getResultList.asScala.toList if (count.nonEmpty) { val head = count.head.asInstanceOf[Array[?]] Some( @@ -147,7 +157,7 @@ class JdbcRepository(entityManagerFactory: EntityManagerFactory) { ) } else None - } + } } def findAll( @@ -158,7 +168,7 @@ class JdbcRepository(entityManagerFactory: EntityManagerFactory) { entityManagerFactory.createEntityManager().runTransactionSync { entityManager => given EntityManager = entityManager - val query1 = entityManager.createNativeQuery(AnnotationSQL.all) + val query1 = entityManager.createNativeQuery(AnnotationSQL.all) limit.foreach(query1.setMaxResults) offset.foreach(query1.setFirstResult) @@ -167,7 +177,7 @@ class JdbcRepository(entityManagerFactory: EntityManagerFactory) { val a2 = executeQueryForAnnotations(a1, includeAncillaryData) entityManager.close() a2 - } + } } diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jdbc/ObservationSQL.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jdbc/ObservationSQL.scala index d262e835..59ddcd90 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jdbc/ObservationSQL.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jdbc/ObservationSQL.scala @@ -78,7 +78,7 @@ object ObservationSQL { // and then sets the parameter for the value def build(sql: String, value: Option[String]): Option[Query] = value.map { v => - val sql2 = sql.replace("(?)", uuidsString) + val sql2 = sql.replace("(?)", uuidsString) val query = entityManager.createNativeQuery(sql2) query.setParameter(1, v) query @@ -87,17 +87,10 @@ object ObservationSQL { // Map of the sql and the value to set. We'll build a query for each value that // is not an Option val params = (updateObserver -> update.observer) - :: (updateGroup -> update.group) - :: (updateConcept -> update.concept) + :: (updateGroup -> update.group) + :: (updateConcept -> update.concept) :: (updateActivity -> update.activity) :: Nil params.flatMap(p => build(p._1, p._2)) - - - - - - - } diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jdbc/TimeHistogramSQL.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jdbc/TimeHistogramSQL.scala index e98b1729..0170a058 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jdbc/TimeHistogramSQL.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jdbc/TimeHistogramSQL.scala @@ -22,7 +22,11 @@ object TimeHistogramSQL { val MinTime = Instant.parse("1987-01-01T00:00:00Z") - def selectFromBinSize(minInstant: Instant, maxInstant: Instant, binSizeDays: Int = 30): String = { + def selectFromBinSize( + minInstant: Instant, + maxInstant: Instant, + binSizeDays: Int = 30 + ): String = { val intervalMillis = binSizeDays * 24 * 60 * 60 * 1000L val start = minInstant.toEpochMilli val end = maxInstant.toEpochMilli diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jpa/CachedVideoReferenceInfoDAOImpl.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jpa/CachedVideoReferenceInfoDAOImpl.scala index af8125d0..5374f772 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jpa/CachedVideoReferenceInfoDAOImpl.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jpa/CachedVideoReferenceInfoDAOImpl.scala @@ -79,7 +79,8 @@ class CachedVideoReferenceInfoDAOImpl(entityManager: EntityManager) private def fetchUsing(namedQuery: String): Iterable[String] = val query = entityManager.createNamedQuery(namedQuery) query.setHint(QueryHints.HINT_READONLY, true) - query.getResultList + query + .getResultList .asScala .filter(_ != null) .map(_.toString) diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jpa/EntityManagerFactories.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jpa/EntityManagerFactories.scala index 68584523..501e9a55 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jpa/EntityManagerFactories.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jpa/EntityManagerFactories.scala @@ -43,14 +43,14 @@ object EntityManagerFactories { // https://juliuskrah.com/tutorial/2017/02/16/getting-started-with-hikaricp-hibernate-and-jpa/ val PRODUCTION_PROPS = Map( - "hibernate.connection.provider_class" -> "org.hibernate.hikaricp.internal.HikariCPConnectionProvider", - "hibernate.hbm2ddl.auto" -> "validate", - "hibernate.hikari.idleTimeout" -> "30000", - "hibernate.jdbc.batch_size" -> "100", - "hibernate.hikari.maximumPoolSize" -> s"${AppConfig.NumberOfVertxWorkers * 2}", // Same as vertx worker pool threads - "hibernate.hikari.minimumIdle" -> "2", - "hibernate.order_inserts" -> "true", - "hibernate.order_updates" -> "true", + "hibernate.connection.provider_class" -> "org.hibernate.hikaricp.internal.HikariCPConnectionProvider", + "hibernate.hbm2ddl.auto" -> "validate", + "hibernate.hikari.idleTimeout" -> "30000", + "hibernate.jdbc.batch_size" -> "100", + "hibernate.hikari.maximumPoolSize" -> s"${AppConfig.NumberOfVertxWorkers * 2}", // Same as vertx worker pool threads + "hibernate.hikari.minimumIdle" -> "2", + "hibernate.order_inserts" -> "true", + "hibernate.order_updates" -> "true", "hibernate.type.java_time_use_direct_jdbc" -> "true" ) diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jpa/ObservationDAOImpl.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jpa/ObservationDAOImpl.scala index 0378f26f..3d8dd2db 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jpa/ObservationDAOImpl.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/jpa/ObservationDAOImpl.scala @@ -165,7 +165,8 @@ class ObservationDAOImpl(entityManager: EntityManager) override def findAllConcepts(): Seq[String] = val query = entityManager.createNamedQuery("Observation.findAllNames") query.setHint(QueryHints.HINT_READONLY, true) - query.getResultList + query + .getResultList .asScala .filter(_ != null) .map(_.toString) @@ -174,7 +175,8 @@ class ObservationDAOImpl(entityManager: EntityManager) override def findAllGroups(): Seq[String] = val query = entityManager.createNamedQuery("Observation.findAllGroups") query.setHint(QueryHints.HINT_READONLY, true) - query.getResultList + query + .getResultList .asScala .filter(_ != null) .map(_.toString) @@ -183,7 +185,8 @@ class ObservationDAOImpl(entityManager: EntityManager) override def findAllActivities(): Seq[String] = val query = entityManager.createNamedQuery("Observation.findAllActivities") query.setHint(QueryHints.HINT_READONLY, true) - query.getResultList + query + .getResultList .asScala .filter(_ != null) .map(_.toString) diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/query/JDBC.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/query/JDBC.scala index e1438e31..01883670 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/query/JDBC.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/query/JDBC.scala @@ -23,21 +23,23 @@ import scala.collection.mutable.ListBuffer import scala.util.Using object JDBC { - case class Metadata(columnName: String, - columnType: String, - columnSize: Int, - columnLabel: String, - columnClassName: String) + case class Metadata( + columnName: String, + columnType: String, + columnSize: Int, + columnLabel: String, + columnClassName: String + ) object Metadata { def fromResultSet(rs: ResultSet): Seq[Metadata] = { val metadata = rs.getMetaData - val n = metadata.getColumnCount + val n = metadata.getColumnCount for - i <- 1 to n - columnName = metadata.getColumnName(i) - columnType = metadata.getColumnTypeName(i) - columnSize = metadata.getColumnDisplaySize(i) - columnLabel = metadata.getColumnLabel(i) + i <- 1 to n + columnName = metadata.getColumnName(i) + columnType = metadata.getColumnTypeName(i) + columnSize = metadata.getColumnDisplaySize(i) + columnLabel = metadata.getColumnLabel(i) columnClassName = metadata.getColumnClassName(i) yield { JDBC.Metadata(columnName, columnType, columnSize, columnLabel, columnClassName) @@ -46,7 +48,6 @@ object JDBC { } } - class JDBC(user: String, password: String, url: String, driver: String) { Class.forName(driver) @@ -57,58 +58,92 @@ class JDBC(user: String, password: String, url: String, driver: String) { java.sql.DriverManager.getConnection(url, user, password) } - def runQuery[T](sql: String, f: ResultSet => T): Either[Throwable, T] = { - Using.Manager(use => - val conn = use(newConnection()) - conn.setReadOnly(true) - val stmt = use(conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY)) - val rs = use(stmt.executeQuery(sql)) - f(rs) - ).toEither + def runQuery[T]( + sql: String, + f: ResultSet => T, + limit: Option[Int] = None, + offset: Option[Int] = None + ): Either[Throwable, T] = { + Using + .Manager(use => + val conn = use(newConnection()) + conn.setReadOnly(true) + val stmt = use( + conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY) + ) + limit.foreach(stmt.setMaxRows) + offset.foreach(stmt.setFetchSize) + val rs = use(stmt.executeQuery(sql)) + f(rs) + ) + .toEither } - def runPreparedStatement(statement: PreparedStatement, f: ResultSet => Unit): Either[Throwable, Unit] = { - Using.Manager(use => - val rs = use(statement.executeQuery()) - f(rs) - ).toEither + def runPreparedStatement( + statement: PreparedStatement, + f: ResultSet => Unit + ): Either[Throwable, Unit] = { + Using + .Manager(use => + val rs = use(statement.executeQuery()) + f(rs) + ) + .toEither } def listColumnsMetadata(viewName: String): Either[Throwable, Seq[JDBC.Metadata]] = { - val sql = s"SELECT * FROM $viewName LIMIT 1" - runQuery(sql, rs => JDBC.Metadata.fromResultSet(rs)) + val sql = s"SELECT * FROM $viewName" + runQuery(sql, rs => JDBC.Metadata.fromResultSet(rs), limit = Some(1)) } - def findDistinct[A](viewName: String, columName: String, transform: Object => Option[A]): Either[Throwable, Seq[A]] = - val sql = s"SELECT DISTINCT $columName FROM $viewName" - runQuery(sql, rs => { - val values = ListBuffer.newBuilder[A] - while (rs.next()) { - transform(rs.getObject(columName)) match - case Some(value) => values += value - case None => () + def findDistinct[A]( + viewName: String, + columName: String, + transform: Object => Option[A] + ): Either[Throwable, Seq[A]] = + val sql = s"SELECT DISTINCT $columName FROM $viewName ORDER BY $columName ASC" + runQuery( + sql, + rs => { + val values = ListBuffer.newBuilder[A] + while (rs.next()) { + transform(rs.getObject(columName)) match + case Some(value) => values += value + case None => () + } + values.result().toSeq } - values.result().toSeq - }) + ) def countDistinct(viewName: String, columnName: String): Either[Throwable, Int] = { val sql = s"SELECT COUNT(DISTINCT $columnName) FROM $viewName" - runQuery(sql, rs => { - rs.next() - rs.getInt(1) - }) + runQuery( + sql, + rs => { + rs.next() + rs.getInt(1) + } + ) } - def findMinMax[A](viewName: String, columnName: String, transform: Object => Option[A]): Either[Throwable, Option[(A, A)]] = { - val sql = s"SELECT MIN($columnName), MAX($columnName) FROM $viewName WHERE $columnName IS NOT NULL" - runQuery(sql, rs => { - rs.next() - val minOpt = transform(rs.getObject(1)) - val maxOpt = transform(rs.getObject(2)) - for { - min <- minOpt - max <- maxOpt - } yield (min, max) - }) + def findMinMax[A]( + viewName: String, + columnName: String, + transform: Object => Option[A] + ): Either[Throwable, Option[(A, A)]] = { + val sql = + s"SELECT MIN($columnName), MAX($columnName) FROM $viewName WHERE $columnName IS NOT NULL" + runQuery( + sql, + rs => { + rs.next() + val minOpt = transform(rs.getObject(1)) + val maxOpt = transform(rs.getObject(2)) + for { + min <- minOpt + max <- maxOpt + } yield (min, max) + } + ) } -} \ No newline at end of file +} diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/query/PreparedStatementGenerator.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/query/PreparedStatementGenerator.scala index 022f4bdf..3104df8e 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/query/PreparedStatementGenerator.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/query/PreparedStatementGenerator.scala @@ -22,7 +22,7 @@ import java.sql.PreparedStatement import scala.util.Try object PreparedStatementGenerator { - val ObservationUuid = "observation_uuid" + val ObservationUuid = "observation_uuid" val ImagedMomentUuid = "imaged_moment_uuid" private val log = System.getLogger(getClass.getName) @@ -35,6 +35,24 @@ object PreparedStatementGenerator { log.atDebug.log(s"Bound ${idx - 1} constraints to prepared statement") }.toEither + def buildPreparedStatementTemplateForCounts( + tableName: String, + constraints: Seq[Constraint], + includeConcurrentObservations: Boolean = false, + includeRelatedAssociations: Boolean = false + ): String = + val where = buildWhereClause( + tableName, + constraints, + includeConcurrentObservations, + includeRelatedAssociations + ) + s""" + |SELECT COUNT(*) + |FROM $tableName + |$where + |""".stripMargin + def buildPreparedStatementTemplate( tableName: String, querySelects: Seq[String], @@ -44,7 +62,12 @@ object PreparedStatementGenerator { ): String = val select = (ObservationUuid +: querySelects).mkString(", ") - val where = buildWhereClause(tableName, constraints, includeConcurrentObservations, includeRelatedAssociations) + val where = buildWhereClause( + tableName, + constraints, + includeConcurrentObservations, + includeRelatedAssociations + ) s""" |SELECT $select @@ -52,12 +75,13 @@ object PreparedStatementGenerator { |WHERE $where |""".stripMargin - - private def buildWhereClause(tableName: String, - constraints: Seq[Constraint], - includeConcurrentObservations: Boolean = false, - includeRelatedAssociations: Boolean = false): String = - val wheres = constraints.map(_.toPreparedStatementTemplate).mkString(" AND ") + private def buildWhereClause( + tableName: String, + constraints: Seq[Constraint], + includeConcurrentObservations: Boolean = false, + includeRelatedAssociations: Boolean = false + ): String = + val wheres = constraints.map(_.toPreparedStatementTemplate).mkString(" AND ") if includeConcurrentObservations && includeRelatedAssociations then s"""WHERE $ObservationUuid IN ( | SELECT $ObservationUuid @@ -69,20 +93,17 @@ object PreparedStatementGenerator { | ) |) |""".stripMargin - else if includeConcurrentObservations then - s"""WHERE $ImagedMomentUuid IN ( + else if includeConcurrentObservations then s"""WHERE $ImagedMomentUuid IN ( | SELECT $ImagedMomentUuid | FROM $tableName | WHERE $wheres |) |""".stripMargin - else if includeRelatedAssociations then - s"""WHERE $ObservationUuid IN ( + else if includeRelatedAssociations then s"""WHERE $ObservationUuid IN ( | SELECT $ObservationUuid | FROM $tableName | WHERE $wheres |) |""".stripMargin - else - s"""WHERE $wheres""" + else s"""WHERE $wheres""" } diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/query/QueryResults.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/query/QueryResults.scala index 62a400b1..adea0d03 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/query/QueryResults.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/query/QueryResults.scala @@ -25,14 +25,26 @@ object QueryResults { def fromResultSet(rs: ResultSet): QueryResults = val metadata = JDBC.Metadata.fromResultSet(rs) - val map = scala.collection.mutable.Map[JDBC.Metadata, mutable.ListBuffer[Any]]() + val map = scala.collection.mutable.Map[JDBC.Metadata, mutable.ListBuffer[Any]]() while rs.next() do metadata.foreach { m => val list = map.getOrElseUpdate(m, mutable.ListBuffer()) list += rs.getObject(m.columnName) } - val data = map.map { case (k, v) => k -> v.result() }.toMap + val data = map.map { case (k, v) => k -> v.result() }.toMap data + def toTsv(queryResults: QueryResults): LazyList[String] = + val header = queryResults.keys.map(_.columnName).mkString("\t") + val values = queryResults.values + val n = values.headOption.map(_.size).getOrElse(0) + LazyList + .tabulate(n) { i => + val s = + for v <- values + yield v(i) + s.mkString("\t") + } + .prepended(header) } diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/query/QueryService.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/query/QueryService.scala index bf30749c..765d28ec 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/query/QueryService.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/query/QueryService.scala @@ -19,16 +19,17 @@ package org.mbari.annosaurus.repository.query import org.mbari.annosaurus.DatabaseConfig import org.mbari.annosaurus.repository.jdbc.* import org.mbari.annosaurus.domain.{Annotation, Association} +import org.mbari.annosaurus.etc.jdk.Logging.{*, given} import java.sql.ResultSet import scala.collection.mutable.ListBuffer import scala.util.Using - class QueryService(databaseConfig: DatabaseConfig, viewName: String) { - private val jdbc = new JDBC(databaseConfig) + val jdbc = new JDBC(databaseConfig) val AnnotationViewName = "annotations" + private val log = System.getLogger(getClass.getName) def findAllConceptNames(): Either[Throwable, Seq[String]] = jdbc.findDistinct(viewName, "concept", stringConverter) @@ -45,45 +46,76 @@ class QueryService(databaseConfig: DatabaseConfig, viewName: String) { | WHERE | concept IN (${concepts.map(s => s"'$s'").mkString(",")}) |""".stripMargin - jdbc.runQuery(sql, rs => { - val associations = ListBuffer.newBuilder[Association] - while (rs.next()) { - val linkName = rs.getString("link_name") - val toConcept = rs.getString("to_concept") - val linkValue = rs.getString("link_value") - associations += Association(linkName, toConcept, linkValue) + jdbc.runQuery( + sql, + rs => { + val associations = ListBuffer.newBuilder[Association] + while (rs.next()) { + val linkName = rs.getString("link_name") + val toConcept = rs.getString("to_concept") + val linkValue = rs.getString("link_value") + associations += Association(linkName, toConcept, linkValue) + } + associations.result().toSeq.sortBy(_.linkName) } - associations.result().toSeq.sortBy(_.linkName) - }) + ) } - def query(querySelects: Seq[String], - constraints: Seq[Constraint], - includeConcurrentObservations: Boolean = false, - includeRelatedAssociations: Boolean = false): Either[Throwable, QueryResults] = + def count(constraints: Constraints): Either[Throwable, Int] = + val sql = PreparedStatementGenerator.buildPreparedStatementTemplateForCounts( + viewName, + constraints.constraints, + constraints.concurrentObservations.getOrElse(false), + constraints.relatedAssociations.getOrElse(false) + ) + log.atDebug.log(s"Running query: $sql") + Using + .Manager(use => + val conn = use(jdbc.newConnection()) + conn.setReadOnly(true) + val stmt = use( + conn.prepareStatement( + sql, + ResultSet.TYPE_FORWARD_ONLY, + ResultSet.CONCUR_READ_ONLY + ) + ) + PreparedStatementGenerator.bind(stmt, constraints.constraints) + val rs = use(stmt.executeQuery(sql)) + if rs.next() then rs.getInt(1) else 0 + ) + .toEither + + def query( + querySelects: Seq[String], + constraints: Constraints + ): Either[Throwable, QueryResults] = val sql = PreparedStatementGenerator.buildPreparedStatementTemplate( viewName, querySelects, - constraints, - includeConcurrentObservations, - includeRelatedAssociations + constraints.constraints, + constraints.concurrentObservations.getOrElse(false), + constraints.relatedAssociations.getOrElse(false) ) - Using.Manager(use => - val conn = use(jdbc.newConnection()) - conn.setReadOnly(true) - val stmt = use(conn.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY)) - PreparedStatementGenerator.bind(stmt, constraints) - val rs = use(stmt.executeQuery(sql)) - QueryResults.fromResultSet(rs) - ).toEither - - - - - - - - + log.atDebug.log(s"Running query: $sql") + Using + .Manager(use => + val conn = use(jdbc.newConnection()) + conn.setReadOnly(true) + val stmt = use( + conn.prepareStatement( + sql, + ResultSet.TYPE_FORWARD_ONLY, + ResultSet.CONCUR_READ_ONLY + ) + ) + constraints.limit.foreach(stmt.setMaxRows) + constraints.offset.foreach(stmt.setFetchSize) + PreparedStatementGenerator.bind(stmt, constraints.constraints) + val rs = use(stmt.executeQuery(sql)) + QueryResults.fromResultSet(rs) + ) + .toEither } diff --git a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/query/constraints.scala b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/query/constraints.scala index fac56731..cfb20203 100644 --- a/annosaurus/src/main/scala/org/mbari/annosaurus/repository/query/constraints.scala +++ b/annosaurus/src/main/scala/org/mbari/annosaurus/repository/query/constraints.scala @@ -21,9 +21,14 @@ import java.time.Instant import java.time.Instant - // Define the Root case class -case class Constraints(constraints: List[Constraint]) +case class Constraints( + constraints: List[Constraint], + limit: Option[Int] = None, + offset: Option[Int] = None, + concurrentObservations: Option[Boolean] = None, + relatedAssociations: Option[Boolean] = None +) sealed trait Constraint: def columnName: String @@ -33,15 +38,28 @@ sealed trait Constraint: def bind(statement: PreparedStatement, idx: Int): Int object Constraint: - object Noop extends Constraint: - val columnName: String = "" - def toPreparedStatementTemplate: String = "" + case object Noop extends Constraint: + val columnName: String = "" + def toPreparedStatementTemplate: String = "" @throws[SQLException] def bind(statement: PreparedStatement, idx: Int): Int = idx + case class Date(columnName: String, startTimestamp: Instant, endTimestamp: Instant) + extends Constraint: + @throws[SQLException] + def bind(statement: PreparedStatement, idx: Int): Int = + statement.setObject(idx, startTimestamp) + statement.setObject(idx + 1, endTimestamp) + idx + 2 + + def toPreparedStatementTemplate: String = columnName + " BETWEEN ? AND ?" + case class In[A](columnName: String, in: Seq[A]) extends Constraint: require(columnName != null) - require(in != null && in.nonEmpty, "Check your value arg! null and empty values are not allowed") + require( + in != null && in.nonEmpty, + "Check your value arg! null and empty values are not allowed" + ) @throws[SQLException] def bind(statement: PreparedStatement, idx: Int): Int = @@ -88,16 +106,12 @@ object Constraint: def toPreparedStatementTemplate: String = columnName + " BETWEEN ? AND ?" - case class Date(columnName: String, startTimestamp: Instant, endTimestamp: Instant) extends Constraint: + case class IsNull(columnName: String, isNull: Boolean) extends Constraint: @throws[SQLException] - def bind(statement: PreparedStatement, idx: Int): Int = - statement.setObject(idx, startTimestamp) - statement.setObject(idx + 1, endTimestamp) - idx + 2 - - def toPreparedStatementTemplate: String = columnName + " BETWEEN ? AND ?" - + def bind(statement: PreparedStatement, idx: Int): Int = idx + def toPreparedStatementTemplate: String = if isNull then columnName + " IS NULL" + else columnName + " IS NOT NULL" // //case class InConstraint[A](columnName: String, in: Seq[A]) extends Constraint { // require(columnName != null) @@ -169,4 +183,3 @@ object Constraint: // def toPreparedStatementTemplate: String = columnName + " BETWEEN ? AND ?" // // - diff --git a/annosaurus/src/test/resources/json/constraints.json b/annosaurus/src/test/resources/json/constraints.json index 33316b8a..731d200f 100644 --- a/annosaurus/src/test/resources/json/constraints.json +++ b/annosaurus/src/test/resources/json/constraints.json @@ -19,6 +19,13 @@ 100 ] }, + { + "columnName": "height", + "between": [ + 1, + 100 + ] + }, { "columnName": "foo", "min": 1 @@ -33,6 +40,16 @@ "2010-01-01T01:22:33Z", "2011-01-01T01:22:33Z" ] + }, + { + "columnName": "qux", + "isnull": true + }, + + { + "columnName": "quxx", + "isnull": false } + ] } \ No newline at end of file diff --git a/annosaurus/src/test/scala/org/mbari/annosaurus/etc/circe/CirceCodecsSuite.scala b/annosaurus/src/test/scala/org/mbari/annosaurus/etc/circe/CirceCodecsSuite.scala index ed5c515d..1f670723 100644 --- a/annosaurus/src/test/scala/org/mbari/annosaurus/etc/circe/CirceCodecsSuite.scala +++ b/annosaurus/src/test/scala/org/mbari/annosaurus/etc/circe/CirceCodecsSuite.scala @@ -312,7 +312,8 @@ class CirceCodecsSuite extends munit.FunSuite { val opt = json.reify[Constraints].toOption assert(opt.isDefined) val constraints = opt.get.constraints - assertEquals(constraints.size, 6) + assertEquals(constraints.size, 9) + println(constraints) } } diff --git a/build.sbt b/build.sbt index 4d817f85..623a6645 100644 --- a/build.sbt +++ b/build.sbt @@ -8,7 +8,7 @@ ThisBuild / licenses := Seq("Apache-2.0" -> url("http://www.apache.org/l ThisBuild / organization := "org.mbari" ThisBuild / organizationName := "Monterey Bay Aquarium Research Institute" ThisBuild / resolvers ++= Seq(Resolver.githubPackages("mbari-org", "maven")) -ThisBuild / scalaVersion := "3.5.0" +ThisBuild / scalaVersion := "3.5.1" // ThisBuild / scalaVersion := "3.3.1" // Fails. See https://github.com/lampepfl/dotty/issues/17069#issuecomment-1763053572 ThisBuild / scalacOptions ++= Seq( "-deprecation", // Emit warning and location for usages of deprecated APIs. diff --git a/it/src/main/scala/org/mbari/annosaurus/controllers/AnnotationControllerSuite.scala b/it/src/main/scala/org/mbari/annosaurus/controllers/AnnotationControllerSuite.scala index c3becfc0..eb0f4f83 100644 --- a/it/src/main/scala/org/mbari/annosaurus/controllers/AnnotationControllerSuite.scala +++ b/it/src/main/scala/org/mbari/annosaurus/controllers/AnnotationControllerSuite.scala @@ -190,8 +190,8 @@ trait AnnotationControllerSuite extends BaseDAOSuite { } test("create(Annotation)") { - val im = TestUtils.build(1, 1).head - val expected = Annotation.from(im.getObservations.asScala.head, true) + val im = TestUtils.build(1, 1).head + val expected = Annotation.from(im.getObservations.asScala.head, true) val obtained = exec(controller.create(expected)).head assert(obtained.observationUuid.isDefined) assert(obtained.imagedMomentUuid.isDefined) diff --git a/it/src/main/scala/org/mbari/annosaurus/endpoints/AnalysisEndpointsSuite.scala b/it/src/main/scala/org/mbari/annosaurus/endpoints/AnalysisEndpointsSuite.scala index dd14a700..539865c8 100644 --- a/it/src/main/scala/org/mbari/annosaurus/endpoints/AnalysisEndpointsSuite.scala +++ b/it/src/main/scala/org/mbari/annosaurus/endpoints/AnalysisEndpointsSuite.scala @@ -17,7 +17,12 @@ package org.mbari.annosaurus.endpoints import org.mbari.annosaurus.controllers.TestUtils -import org.mbari.annosaurus.domain.{DepthHistogramSC, QueryConstraints, QueryConstraintsResponseSC, TimeHistogramSC} +import org.mbari.annosaurus.domain.{ + DepthHistogramSC, + QueryConstraints, + QueryConstraintsResponseSC, + TimeHistogramSC +} import sttp.tapir.* import sttp.client3.* import sttp.model.StatusCode @@ -78,23 +83,22 @@ trait AnalysisEndpointsSuite extends EndpointsSuite { // TODO both the depth and time histogram logic needs to be reworked. They can give incorrect results test("timeHistogram".flaky) { - val xs = TestUtils.build(10, 10, includeData = true) + val xs = TestUtils.build(10, 10, includeData = true) val start = Instant.parse("1888-08-01T00:00:00Z") - for - i <- xs.indices - do - xs(i).setRecordedTimestamp(start.plusSeconds(i * 60 * 60)) - val dao = daoFactory.newImagedMomentDAO() + for i <- xs.indices + do xs(i).setRecordedTimestamp(start.plusSeconds(i * 60 * 60)) + val dao = daoFactory.newImagedMomentDAO() dao.runTransaction(d => xs.foreach(d.create)) - val minTime = xs.map(_.getRecordedTimestamp).min - val maxTime = xs.map(_.getRecordedTimestamp).max + val minTime = xs.map(_.getRecordedTimestamp).min + val maxTime = xs.map(_.getRecordedTimestamp).max val expected = xs.filter(_.getRecordedTimestamp != null).flatMap(_.getObservations.asScala).size val videoReferenceUuids = xs.map(_.getVideoReferenceUuid).distinct - val qcr = QueryConstraints( + val qcr = QueryConstraints( videoReferenceUuids = Seq(xs.head.getVideoReferenceUuid), minTimestamp = Some(minTime.minusSeconds(24 * 60 * 60)), - maxTimestamp = Some(maxTime.plusSeconds(24 * 60 * 60))) + maxTimestamp = Some(maxTime.plusSeconds(24 * 60 * 60)) + ) // println(qcr.toSnakeCase.stringify) runPost( endpoints.timeHistogramImpl, diff --git a/it/src/main/scala/org/mbari/annosaurus/endpoints/AnnotationEndpointsSuite.scala b/it/src/main/scala/org/mbari/annosaurus/endpoints/AnnotationEndpointsSuite.scala index cbe6556a..c873bc94 100644 --- a/it/src/main/scala/org/mbari/annosaurus/endpoints/AnnotationEndpointsSuite.scala +++ b/it/src/main/scala/org/mbari/annosaurus/endpoints/AnnotationEndpointsSuite.scala @@ -17,7 +17,15 @@ package org.mbari.annosaurus.endpoints import org.mbari.annosaurus.controllers.{AnnotationController, TestUtils} -import org.mbari.annosaurus.domain.{Annotation, AnnotationCreate, AnnotationSC, ConcurrentRequest, ConcurrentRequestCountSC, MultiRequest, MultiRequestCountSC} +import org.mbari.annosaurus.domain.{ + Annotation, + AnnotationCreate, + AnnotationSC, + ConcurrentRequest, + ConcurrentRequestCountSC, + MultiRequest, + MultiRequestCountSC +} import org.mbari.annosaurus.repository.jpa.JPADAOFactory import org.mbari.annosaurus.etc.jdk.Logging.{*, given} import org.mbari.annosaurus.etc.jwt.JwtService @@ -503,17 +511,17 @@ trait AnnotationEndpointsSuite extends EndpointsSuite { } test("create (stress test)") { - val count = new AtomicInteger(0) - val jwt = jwtService.authorize("foo").orNull - val uuid = UUID.randomUUID() + val count = new AtomicInteger(0) + val jwt = jwtService.authorize("foo").orNull + val uuid = UUID.randomUUID() assert(jwt != null) val backendStub = newBackendStub(endpoints.createAnnotationImpl) - val n = 1000 - var threads = (0 until n) + val n = 1000 + var threads = (0 until n) .map(i => TestUtils.build(1, 1).head) .map(im => { val obs = im.getObservations.iterator.next() - val a = Annotation.from(obs) + val a = Annotation.from(obs) a.copy(videoReferenceUuid = Some(uuid), timecode = None) }) .map(anno => { @@ -543,6 +551,5 @@ trait AnnotationEndpointsSuite extends EndpointsSuite { Thread.sleep(100) } - } } diff --git a/it/src/main/scala/org/mbari/annosaurus/endpoints/ImagedMomentEndpointsSuite.scala b/it/src/main/scala/org/mbari/annosaurus/endpoints/ImagedMomentEndpointsSuite.scala index d12caf38..5b874f20 100644 --- a/it/src/main/scala/org/mbari/annosaurus/endpoints/ImagedMomentEndpointsSuite.scala +++ b/it/src/main/scala/org/mbari/annosaurus/endpoints/ImagedMomentEndpointsSuite.scala @@ -591,7 +591,7 @@ trait ImagedMomentEndpointsSuite extends EndpointsSuite { assertEquals(response.code, StatusCode.Ok) val count = checkResponse[Count](response.body) assertEquals(count.count.intValue, xs.size) - + } test("bulkMove (snake_case)") { @@ -612,6 +612,6 @@ trait ImagedMomentEndpointsSuite extends EndpointsSuite { assertEquals(response.code, StatusCode.Ok) val count = checkResponse[Count](response.body) assertEquals(count.count.intValue, xs.size) - + } } diff --git a/it/src/main/scala/org/mbari/annosaurus/endpoints/ObservationEndpointsSuite.scala b/it/src/main/scala/org/mbari/annosaurus/endpoints/ObservationEndpointsSuite.scala index 493a8eae..967c946c 100644 --- a/it/src/main/scala/org/mbari/annosaurus/endpoints/ObservationEndpointsSuite.scala +++ b/it/src/main/scala/org/mbari/annosaurus/endpoints/ObservationEndpointsSuite.scala @@ -18,7 +18,18 @@ package org.mbari.annosaurus.endpoints import org.mbari.annosaurus.Endpoints.daoFactory import org.mbari.annosaurus.controllers.{ImagedMomentController, ObservationController, TestUtils} -import org.mbari.annosaurus.domain.{ConceptCount, Count, CountForVideoReferenceSC, ImagedMoment, Observation, ObservationSC, ObservationUpdateSC, ObservationsUpdate, RenameConcept, RenameCountSC} +import org.mbari.annosaurus.domain.{ + ConceptCount, + Count, + CountForVideoReferenceSC, + ImagedMoment, + Observation, + ObservationSC, + ObservationUpdateSC, + ObservationsUpdate, + RenameConcept, + RenameCountSC +} import org.mbari.annosaurus.etc.jwt.JwtService import org.mbari.annosaurus.repository.jpa.JPADAOFactory import org.mbari.annosaurus.etc.tapir.TapirCodecs.given @@ -44,11 +55,10 @@ trait ObservationEndpointsSuite extends EndpointsSuite { given JPADAOFactory = daoFactory - given jwtService: JwtService = new JwtService("mbari", "foo", "bar") - private lazy val controller = ObservationController(daoFactory) - private lazy val jdbcRepository = new JdbcRepository(daoFactory.entityManagerFactory) - private lazy val endpoints = new ObservationEndpoints(controller, jdbcRepository) - + given jwtService: JwtService = new JwtService("mbari", "foo", "bar") + private lazy val controller = ObservationController(daoFactory) + private lazy val jdbcRepository = new JdbcRepository(daoFactory.entityManagerFactory) + private lazy val endpoints = new ObservationEndpoints(controller, jdbcRepository) test("findObservationByUuid") { val im = TestUtils.create(1, 1).head @@ -509,12 +519,13 @@ trait ObservationEndpointsSuite extends EndpointsSuite { test("updateManyObservations") { val im = TestUtils.create(20, 2) val observationUuids = im.flatMap(_.getObservations.asScala.map(_.getUuid)).toSeq - val update = ObservationsUpdate( + val update = ObservationsUpdate( observationUuids, concept = Some("new-concept"), observer = Some("new-observer"), activity = Some("new-activity"), - group = Some("new-group")) + group = Some("new-group") + ) val jwt = jwtService.authorize("foo").orNull assert(jwt != null) val backend = newBackendStub(endpoints.updateManyObservationsImpl) @@ -526,13 +537,12 @@ trait ObservationEndpointsSuite extends EndpointsSuite { .body(update.stringify) .send(backend) .join - val obtained = checkResponse[Count](response.body) + val obtained = checkResponse[Count](response.body) assertEquals(obtained.count, observationUuids.size.toLong) - for - uuid <- observationUuids + for uuid <- observationUuids do - val opt = controller.findByUUID(uuid).join + val opt = controller.findByUUID(uuid).join assert(opt.isDefined) val obtained = opt.get assertEquals(obtained.concept, update.concept.get) diff --git a/it/src/main/scala/org/mbari/annosaurus/repository/jdbc/AnalysisRepositorySuite.scala b/it/src/main/scala/org/mbari/annosaurus/repository/jdbc/AnalysisRepositorySuite.scala index dace3641..07095ec1 100644 --- a/it/src/main/scala/org/mbari/annosaurus/repository/jdbc/AnalysisRepositorySuite.scala +++ b/it/src/main/scala/org/mbari/annosaurus/repository/jdbc/AnalysisRepositorySuite.scala @@ -41,22 +41,21 @@ trait AnalysisRepositorySuite extends BaseDAOSuite { } test("timeHistogram") { - val xs = TestUtils.build(5, 5, includeData = true) + val xs = TestUtils.build(5, 5, includeData = true) val start = Instant.parse("1888-08-01T00:00:00Z") - for - i <- xs.indices - do - xs(i).setRecordedTimestamp(start.plusSeconds(i * 60 * 60)) - val dao = daoFactory.newImagedMomentDAO() + for i <- xs.indices + do xs(i).setRecordedTimestamp(start.plusSeconds(i * 60 * 60)) + val dao = daoFactory.newImagedMomentDAO() dao.runTransaction(d => xs.foreach(d.create)) - val minTime = xs.map(_.getRecordedTimestamp).min + val minTime = xs.map(_.getRecordedTimestamp).min val maxTime = xs.map(_.getRecordedTimestamp).max // xs.foreach(x => println(ImagedMoment.from(x, true).stringify)) val expected = xs.flatMap(_.getObservations.asScala).size val qcr = QueryConstraints( videoReferenceUuids = Seq(xs.head.getVideoReferenceUuid), minTimestamp = Some(minTime.minusSeconds(24 * 60 * 60)), - maxTimestamp = Some(maxTime.plusSeconds(24 * 60 * 60))) + maxTimestamp = Some(maxTime.plusSeconds(24 * 60 * 60)) + ) val histogram = repository.timeHistogram(qcr, 1) // println(histogram) assertEquals(histogram.count, expected) diff --git a/it/src/main/scala/org/mbari/annosaurus/repository/jdbc/JdbcRepositorySuite.scala b/it/src/main/scala/org/mbari/annosaurus/repository/jdbc/JdbcRepositorySuite.scala index 35341c97..1524be82 100644 --- a/it/src/main/scala/org/mbari/annosaurus/repository/jdbc/JdbcRepositorySuite.scala +++ b/it/src/main/scala/org/mbari/annosaurus/repository/jdbc/JdbcRepositorySuite.scala @@ -22,7 +22,13 @@ import org.mbari.annosaurus.repository.jpa.JPADAOFactory import junit.framework.Test import scala.jdk.CollectionConverters.* -import org.mbari.annosaurus.domain.{Annotation, ConcurrentRequest, MultiRequest, ObservationsUpdate, QueryConstraints} +import org.mbari.annosaurus.domain.{ + Annotation, + ConcurrentRequest, + MultiRequest, + ObservationsUpdate, + QueryConstraints +} import java.time.Duration import org.mbari.annosaurus.etc.circe.CirceCodecs.{*, given} @@ -327,15 +333,15 @@ trait JdbcRepositorySuite extends BaseDAOSuite { } test("updateObservations") { - val xs = TestUtils.create(8, 1) + val xs = TestUtils.create(8, 1) val observationUuids = xs.map(im => im.getObservations.asScala.head.getUuid()) val update0 = ObservationsUpdate(observationUuids) - val n = repository.updateObservations(update0) + val n = repository.updateObservations(update0) assertEquals(n, 0) def runUpdate(update: ObservationsUpdate): Unit = { - val n = repository.updateObservations(update) + val n = repository.updateObservations(update) assertEquals(n, xs.size) val dao = daoFactory.newObservationDAO() for (uuid <- observationUuids) { @@ -358,22 +364,21 @@ trait JdbcRepositorySuite extends BaseDAOSuite { val update2 = ObservationsUpdate(observationUuids, observer = Some("observer-foo")) runUpdate(update2) - val update3 = ObservationsUpdate(observationUuids, group = Some("group-foo")) runUpdate(update3) val update4 = ObservationsUpdate(observationUuids, activity = Some("activity-foo")) runUpdate(update4) - val update5 = ObservationsUpdate(observationUuids, + val update5 = ObservationsUpdate( + observationUuids, observer = Some("observer-foo2"), concept = Some("concept-foo2"), group = Some("group-foo2"), - activity = Some("activity-foo2")) + activity = Some("activity-foo2") + ) runUpdate(update5) - - } } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index bbce7510..93d5ca4b 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -30,7 +30,7 @@ object Dependencies { lazy val hibernateEnvers = "org.hibernate.orm" % "hibernate-envers" % hibernateVersion lazy val hibernateHikari = "org.hibernate.orm" % "hibernate-hikaricp" % hibernateVersion - lazy val hikariCp = "com.zaxxer" % "HikariCP" % "5.1.0" + lazy val hikariCp = "com.zaxxer" % "HikariCP" % "6.0.0" lazy val jansi = "org.fusesource.jansi" % "jansi" % "2.4.1" lazy val javaxServlet = "javax.servlet" % "javax.servlet-api" % "4.0.1" lazy val javaxTransaction = "javax.transaction" % "jta" % "1.1" @@ -49,7 +49,7 @@ object Dependencies { lazy val json4sJackson = "org.json4s" %% "json4s-jackson" % "4.0.7" lazy val junit = "junit" % "junit" % "4.13.2" - val logbackVersion = "1.5.10" + val logbackVersion = "1.5.11" lazy val logbackClassic = "ch.qos.logback" % "logback-classic" % logbackVersion lazy val logbackCore = "ch.qos.logback" % "logback-core" % logbackVersion