diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/factory/NussknackerAppFactory.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/factory/NussknackerAppFactory.scala index fe61a39af45..207ee5c1271 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/factory/NussknackerAppFactory.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/factory/NussknackerAppFactory.scala @@ -39,7 +39,13 @@ class NussknackerAppFactory(processingTypeDataStateFactory: ProcessingTypeDataSt db <- DbRef.create(config.resolved) feStatisticsRepository <- QuestDbFEStatisticsRepository.create(system, clock, config.resolved) server = new NussknackerHttpServer( - new AkkaHttpBasedRouteProvider(db, metricsRegistry, processingTypeDataStateFactory, feStatisticsRepository)( + new AkkaHttpBasedRouteProvider( + db, + metricsRegistry, + processingTypeDataStateFactory, + feStatisticsRepository, + clock + )( system, materializer ), diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala index 1a66a18d41f..ca72f8d47d4 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala @@ -99,7 +99,8 @@ class AkkaHttpBasedRouteProvider( dbRef: DbRef, metricsRegistry: MetricRegistry, processingTypeDataStateFactory: ProcessingTypeDataStateFactory, - feStatisticsRepository: FEStatisticsRepository[Future] + feStatisticsRepository: FEStatisticsRepository[Future], + designerClock: Clock )(implicit system: ActorSystem, materializer: Materializer) extends RouteProvider[Route] with Directives @@ -494,6 +495,7 @@ class AkkaHttpBasedRouteProvider( .values .flatten .toList, + designerClock ) val statisticsApiHttpService = new StatisticsApiHttpService( diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/statistics/CorrelationId.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/statistics/CorrelationId.scala new file mode 100644 index 00000000000..e0ccbd4af84 --- /dev/null +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/statistics/CorrelationId.scala @@ -0,0 +1,10 @@ +package pl.touk.nussknacker.ui.statistics + +import java.util.UUID + +class CorrelationId(val value: String) extends AnyVal + +object CorrelationId { + def apply(): CorrelationId = + new CorrelationId(UUID.randomUUID().toString) +} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/statistics/ScenarioStatistics.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/statistics/ScenarioStatistics.scala index 46206104586..32781338532 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/statistics/ScenarioStatistics.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/statistics/ScenarioStatistics.scala @@ -1,7 +1,7 @@ package pl.touk.nussknacker.ui.statistics import cats.implicits.toFoldableOps -import pl.touk.nussknacker.engine.api.component.{BuiltInComponentId, ComponentType, ProcessingMode} +import pl.touk.nussknacker.engine.api.component.{ComponentType, ProcessingMode} import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.definition.component.ComponentDefinitionWithImplementation import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap @@ -26,16 +26,64 @@ object ScenarioStatistics { private val componentStatisticPrefix = "c_" + private[statistics] val emptyScenarioStatistics: Map[String, String] = Map( + ScenarioCount -> 0, + FragmentCount -> 0, + UnboundedStreamCount -> 0, + BoundedStreamCount -> 0, + RequestResponseCount -> 0, + FlinkDMCount -> 0, + LiteK8sDMCount -> 0, + LiteEmbeddedDMCount -> 0, + UnknownDMCount -> 0, + ActiveScenarioCount -> 0 + ).map { case (k, v) => (k.toString, v.toString) } + + private[statistics] val emptyActivityStatistics: Map[String, String] = Map( + AttachmentsAverage -> 0, + AttachmentsTotal -> 0, + CommentsTotal -> 0, + CommentsAverage -> 0 + ).map { case (k, v) => (k.toString, v.toString) } + + private[statistics] val emptyComponentStatistics: Map[String, String] = + Map(ComponentsCount.toString -> "0") + + private[statistics] val emptyUptimeStats: Map[String, String] = Map( + UptimeInSecondsAverage -> 0, + UptimeInSecondsMax -> 0, + UptimeInSecondsMin -> 0, + ).map { case (k, v) => (k.toString, v.toString) } + + private[statistics] val emptyGeneralStatistics: Map[String, String] = Map( + NodesMedian -> 0, + NodesAverage -> 0, + NodesMax -> 0, + NodesMin -> 0, + CategoriesCount -> 0, + VersionsMedian -> 0, + VersionsAverage -> 0, + VersionsMax -> 0, + VersionsMin -> 0, + AuthorsCount -> 0, + FragmentsUsedMedian -> 0, + FragmentsUsedAverage -> 0, + UptimeInSecondsAverage -> 0, + UptimeInSecondsMax -> 0, + UptimeInSecondsMin -> 0, + ).map { case (k, v) => (k.toString, v.toString) } + def getScenarioStatistics(scenariosInputData: List[ScenarioStatisticsInputData]): Map[String, String] = { - scenariosInputData - .map(ScenarioStatistics.determineStatisticsForScenario) - .combineAll - .mapValuesNow(_.toString) + emptyScenarioStatistics ++ + scenariosInputData + .map(ScenarioStatistics.determineStatisticsForScenario) + .combineAll + .mapValuesNow(_.toString) } def getGeneralStatistics(scenariosInputData: List[ScenarioStatisticsInputData]): Map[String, String] = { if (scenariosInputData.isEmpty) { - Map.empty + emptyGeneralStatistics } else { // Nodes stats val sortedNodes = scenariosInputData.map(_.nodesCount).sorted @@ -64,21 +112,16 @@ object ScenarioStatistics { }.sorted val uptimeStatsMap = { if (sortedUptimes.isEmpty) { - Map( - UptimeInSecondsAverage -> 0, - UptimeInSecondsMax -> 0, - UptimeInSecondsMin -> 0, - ) + emptyUptimeStats } else { Map( UptimeInSecondsAverage -> calculateAverage(sortedUptimes), UptimeInSecondsMax -> getMax(sortedUptimes), UptimeInSecondsMin -> getMin(sortedUptimes) - ) + ).map { case (k, v) => (k.toString, v.toString) } } } - - (Map( + Map( NodesMedian -> nodesMedian, NodesAverage -> nodesAverage, NodesMax -> nodesMax, @@ -91,8 +134,9 @@ object ScenarioStatistics { AuthorsCount -> authorsCount, FragmentsUsedMedian -> fragmentsUsedMedian, FragmentsUsedAverage -> fragmentsUsedAverage - ) ++ uptimeStatsMap) - .map { case (k, v) => (k.toString, v.toString) } + ) + .map { case (k, v) => (k.toString, v.toString) } ++ + uptimeStatsMap } } @@ -100,7 +144,7 @@ object ScenarioStatistics { listOfActivities: List[DbProcessActivityRepository.ProcessActivity] ): Map[String, String] = { if (listOfActivities.isEmpty) { - Map.empty + emptyActivityStatistics } else { // Attachment stats val sortedAttachmentCountList = listOfActivities.map(_.attachments.length) @@ -126,7 +170,7 @@ object ScenarioStatistics { components: List[ComponentDefinitionWithImplementation] ): Map[String, String] = { if (componentList.isEmpty) { - Map.empty + emptyComponentStatistics } else { // Get number of available components to check how many custom components created @@ -204,15 +248,15 @@ object ScenarioStatistics { private def getMax[T: Numeric](orderedList: List[T]): T = { if (orderedList.isEmpty) implicitly[Numeric[T]].zero - else orderedList.head + else orderedList.last } private def getMin[T: Numeric](orderedList: List[T]): T = { if (orderedList.isEmpty) implicitly[Numeric[T]].zero - else orderedList.last + else orderedList.head } - def mapNameToStat(componentId: String): String = { + private def mapNameToStat(componentId: String): String = { val shortenedName = componentId.replaceAll(vowelsRegex, "").toLowerCase componentStatisticPrefix + shortenedName @@ -254,6 +298,9 @@ case object LiteK8sDMCount extends StatisticKey("s_dm_l") case object LiteEmbeddedDMCount extends StatisticKey("s_dm_e") case object UnknownDMCount extends StatisticKey("s_dm_c") case object ActiveScenarioCount extends StatisticKey("s_a") -case object NuSource extends StatisticKey("source") // f.e docker, helmchart, docker-quickstart, binaries -case object NuFingerprint extends StatisticKey("fingerprint") -case object NuVersion extends StatisticKey("version") +// Not scenario related statistics +case object NuSource extends StatisticKey("source") // f.e docker, helmchart, docker-quickstart, binaries +case object NuFingerprint extends StatisticKey("fingerprint") +case object NuVersion extends StatisticKey("version") +case object CorrelationIdStat extends StatisticKey("co_id") +case object DesignerUptimeInSeconds extends StatisticKey("d_u") diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/statistics/StatisticsUrls.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/statistics/StatisticsUrls.scala index 5009c283ffd..9c07b3ac515 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/statistics/StatisticsUrls.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/statistics/StatisticsUrls.scala @@ -8,13 +8,17 @@ import java.nio.charset.StandardCharsets class StatisticsUrls(cfg: StatisticUrlConfig) extends LazyLogging { - def prepare(fingerprint: Fingerprint, rawStatistics: Map[String, String]): List[String] = + def prepare( + fingerprint: Fingerprint, + correlationId: CorrelationId, + rawStatistics: Map[String, String] + ): List[String] = rawStatistics.toList // Sorting for purpose of easier testing .sortBy(_._1) .map(encodeQueryParam) .groupByMaxChunkSize(cfg.urlBytesSizeLimit) - .flatMap(queryParams => prepareUrlString(queryParams, queryParamsForEveryURL(fingerprint))) + .flatMap(queryParams => prepareUrlString(queryParams, queryParamsForEveryURL(fingerprint, correlationId))) private def encodeQueryParam(entry: (String, String)): String = s"${URLEncoder.encode(entry._1, StandardCharsets.UTF_8)}=${URLEncoder.encode(entry._2, StandardCharsets.UTF_8)}" @@ -28,8 +32,9 @@ class StatisticsUrls(cfg: StatisticUrlConfig) extends LazyLogging { } } - private def queryParamsForEveryURL(fingerprint: Fingerprint): List[String] = List( - encodeQueryParam(NuFingerprint.name -> fingerprint.value) + private def queryParamsForEveryURL(fingerprint: Fingerprint, correlationId: CorrelationId): List[String] = List( + encodeQueryParam(NuFingerprint.name -> fingerprint.value), + encodeQueryParam(CorrelationIdStat.name -> correlationId.value) ) } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/statistics/UsageStatisticsReportsSettingsService.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/statistics/UsageStatisticsReportsSettingsService.scala index d780a7a0dbf..13e999a4ed5 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/statistics/UsageStatisticsReportsSettingsService.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/statistics/UsageStatisticsReportsSettingsService.scala @@ -21,6 +21,7 @@ import pl.touk.nussknacker.ui.process.{ProcessService, ScenarioQuery} import pl.touk.nussknacker.ui.security.api.{LoggedUser, NussknackerInternalUser} import pl.touk.nussknacker.ui.statistics.UsageStatisticsReportsSettingsService.nuFingerprintFileName +import java.time.Clock import scala.concurrent.{ExecutionContext, Future} object UsageStatisticsReportsSettingsService extends LazyLogging { @@ -38,7 +39,8 @@ object UsageStatisticsReportsSettingsService extends LazyLogging { // TODO: Should not depend on DTO, need to extract usageCount and check if all available components are present using processingTypeDataProvider componentService: ComponentService, statisticsRepository: FEStatisticsRepository[Future], - componentList: List[ComponentDefinitionWithImplementation] + componentList: List[ComponentDefinitionWithImplementation], + designerClock: Clock )(implicit ec: ExecutionContext): UsageStatisticsReportsSettingsService = { val ignoringErrorsFEStatisticsRepository = new IgnoringErrorsFEStatisticsRepository(statisticsRepository) implicit val user: LoggedUser = NussknackerInternalUser.instance @@ -93,7 +95,8 @@ object UsageStatisticsReportsSettingsService extends LazyLogging { fetchActivity, fetchComponentList, () => ignoringErrorsFEStatisticsRepository.read(), - componentList + componentList, + designerClock ) } @@ -121,16 +124,21 @@ class UsageStatisticsReportsSettingsService( ], fetchComponentList: () => Future[Either[StatisticError, List[ComponentListElement]]], fetchFeStatistics: () => Future[Map[String, Long]], - components: List[ComponentDefinitionWithImplementation] + components: List[ComponentDefinitionWithImplementation], + designerClock: Clock )(implicit ec: ExecutionContext) { - private val statisticsUrls = new StatisticsUrls(urlConfig) + private val statisticsUrls = new StatisticsUrls(urlConfig) + private val designerStartTime = designerClock.instant() def prepareStatisticsUrl(): Future[Either[StatisticError, List[String]]] = { if (config.enabled) { val maybeUrls = for { queryParams <- determineQueryParams() fingerprint <- new EitherT(fingerprintService.fingerprint(config, nuFingerprintFileName)) - urls <- EitherT.pure[Future, StatisticError](statisticsUrls.prepare(fingerprint, queryParams)) + correlationId = CorrelationId.apply() + urls <- EitherT.pure[Future, StatisticError]( + statisticsUrls.prepare(fingerprint, correlationId, queryParams) + ) } yield urls maybeUrls.value } else { @@ -149,11 +157,13 @@ class UsageStatisticsReportsSettingsService( componentList <- new EitherT(fetchComponentList()) componentStatistics = ScenarioStatistics.getComponentStatistic(componentList, components) feStatistics <- EitherT.liftF(fetchFeStatistics()) + designerUptimeStatistics = getDesignerUptimeStatistics } yield basicStatistics ++ scenariosStatistics ++ generalStatistics ++ activityStatistics ++ componentStatistics ++ + designerUptimeStatistics ++ feStatistics.map { case (k, v) => k -> v.toString } @@ -168,6 +178,14 @@ class UsageStatisticsReportsSettingsService( NuVersion.name -> BuildInfo.version ) + private def getDesignerUptimeStatistics: Map[String, String] = { + Map( + DesignerUptimeInSeconds.name -> (designerClock + .instant() + .getEpochSecond - designerStartTime.getEpochSecond).toString + ) + } + } private[statistics] case class ScenarioStatisticsInputData( diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/StatisticsApiHttpServiceBusinessSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/StatisticsApiHttpServiceBusinessSpec.scala index dbfadeecce3..ae1a5f6b60e 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/StatisticsApiHttpServiceBusinessSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/StatisticsApiHttpServiceBusinessSpec.scala @@ -70,6 +70,7 @@ class StatisticsApiHttpServiceBusinessSpec private val statisticsByIndex = statisticsNames.zipWithIndex.map(p => p._2 -> p._1).toMap private val quote = '"' private val random = new Random() + private val uuidRegex = "[0-9a-f]{8}(-[a-f0-9]{4}){4}[a-f0-9]{8}" private val mockedClock = mock[Clock](new Answer[Instant] { override def answer(invocation: InvocationOnMock): Instant = Instant.now() @@ -97,10 +98,41 @@ class StatisticsApiHttpServiceBusinessSpec .Then() .statusCode(200) .bodyWithStatisticsURL( + (AuthorsCount.name, equalTo("0")), + (AttachmentsTotal.name, equalTo("0")), + (AttachmentsAverage.name, equalTo("0")), + (CategoriesCount.name, equalTo("0")), (ComponentsCount.name, new GreaterThanOrEqualToLongMatcher(62L)), + (CommentsTotal.name, equalTo("0")), + (CommentsAverage.name, equalTo("0")), + (FragmentsUsedMedian.name, equalTo("0")), + (FragmentsUsedAverage.name, equalTo("0")), (NuFingerprint.name, matchesRegex("[\\w-]+?")), + (NodesMedian.name, equalTo("0")), + (NodesMax.name, equalTo("0")), + (NodesMin.name, equalTo("0")), + (NodesAverage.name, equalTo("0")), + (ActiveScenarioCount.name, equalTo("0")), + (UnknownDMCount.name, equalTo("0")), + (LiteEmbeddedDMCount.name, equalTo("0")), + (FlinkDMCount.name, equalTo("0")), + (LiteK8sDMCount.name, equalTo("0")), + (FragmentCount.name, equalTo("0")), + (BoundedStreamCount.name, equalTo("0")), + (RequestResponseCount.name, equalTo("0")), + (UnboundedStreamCount.name, equalTo("0")), + (ScenarioCount.name, equalTo("0")), (NuSource.name, equalTo("sources")), + (UptimeInSecondsMax.name, equalTo("0")), + (UptimeInSecondsMin.name, equalTo("0")), + (UptimeInSecondsAverage.name, equalTo("0")), + (VersionsMedian.name, equalTo("0")), + (VersionsMax.name, equalTo("0")), + (VersionsMin.name, equalTo("0")), + (VersionsAverage.name, equalTo("0")), (NuVersion.name, equalTo(nuVersion)), + (CorrelationIdStat.name, matchesRegex(uuidRegex)), + (DesignerUptimeInSeconds.name, matchesRegex("\\d+")) ) } @@ -148,6 +180,8 @@ class StatisticsApiHttpServiceBusinessSpec (VersionsMin.name, equalTo("1")), (VersionsAverage.name, equalTo("1")), (NuVersion.name, equalTo(nuVersion)), + (CorrelationIdStat.name, matchesRegex(uuidRegex)), + (DesignerUptimeInSeconds.name, matchesRegex("\\d+")), // TODO: Should make a proper test for component mapping ("c_bltnfltr", equalTo("1")) ) diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/statistics/ScenarioStatisticsTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/statistics/ScenarioStatisticsTest.scala index b8ff18ae894..82a71c09018 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/statistics/ScenarioStatisticsTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/statistics/ScenarioStatisticsTest.scala @@ -27,9 +27,16 @@ import pl.touk.nussknacker.ui.config.UsageStatisticsReportsConfig import pl.touk.nussknacker.ui.process.processingtype.DeploymentManagerType import pl.touk.nussknacker.ui.process.repository.DbProcessActivityRepository import pl.touk.nussknacker.ui.process.repository.DbProcessActivityRepository.ProcessActivity +import pl.touk.nussknacker.ui.statistics.ScenarioStatistics.{ + emptyActivityStatistics, + emptyComponentStatistics, + emptyGeneralStatistics, + emptyScenarioStatistics, + emptyUptimeStats +} import java.net.URI -import java.time.Instant +import java.time.{Clock, Instant} import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future @@ -49,6 +56,56 @@ class ScenarioStatisticsTest } ) + private val clock: Clock = Clock.systemUTC() + + private val uuidRegex = "[0-9a-f]{8}(-[a-f0-9]{4}){4}[a-f0-9]{8}" + + private val emptyScenarioRelatedStatistics = + emptyScenarioStatistics ++ emptyComponentStatistics ++ emptyActivityStatistics ++ emptyUptimeStats ++ emptyGeneralStatistics + + private val allScenarioRelatedStatistics = Map( + AuthorsCount -> 0, + CategoriesCount -> 0, + ComponentsCount -> 0, + VersionsMedian -> 0, + AttachmentsTotal -> 0, + AttachmentsAverage -> 0, + VersionsMax -> 0, + VersionsMin -> 0, + VersionsAverage -> 0, + UptimeInSecondsAverage -> 0, + UptimeInSecondsMax -> 0, + UptimeInSecondsMin -> 0, + CommentsAverage -> 0, + CommentsTotal -> 0, + FragmentsUsedMedian -> 0, + FragmentsUsedAverage -> 0, + NodesMedian -> 0, + NodesAverage -> 0, + NodesMax -> 0, + NodesMin -> 0, + ScenarioCount -> 0, + FragmentCount -> 0, + UnboundedStreamCount -> 0, + BoundedStreamCount -> 0, + RequestResponseCount -> 0, + FlinkDMCount -> 0, + LiteK8sDMCount -> 0, + LiteEmbeddedDMCount -> 0, + UnknownDMCount -> 0, + ActiveScenarioCount -> 0, + ).map { case (k, v) => (k.toString, v.toString) } + + // This statistics are added in UsageStatisticsReportsSettingsService + // Fingerprint and CorrelationId is added after `determineQueryParams` + private val notScenarioRelatedStatistics = Map( + NuSource -> 0, + NuVersion -> 0, + DesignerUptimeInSeconds -> 0, + // NuFingerprint -> 0, + // CorrelationIdStat -> 0, + ).map { case (k, v) => (k.toString, v.toString) } + test("should determine statistics for running scenario with streaming processing mode and flink engine") { val scenarioData = ScenarioStatisticsInputData( isFragment = false, @@ -214,12 +271,14 @@ class ScenarioStatisticsTest _ => Future.successful(Right(List.empty)), () => Future.successful(Right(List.empty)), () => Future.successful(Map.empty[String, Long]), - List.empty + List.empty, + clock ).prepareStatisticsUrl().futureValue.value urlStrings.length shouldEqual 1 val urlString = urlStrings.head urlString should include(s"fingerprint=$sampleFingerprint") + urlString should include regex s"$CorrelationIdStat=$uuidRegex" urlString should include("source=sources") urlString should include(s"version=${BuildInfo.version}") } @@ -233,7 +292,8 @@ class ScenarioStatisticsTest _ => Future.successful(Right(List.empty)), () => Future.successful(Right(componentList)), () => Future.successful(Map.empty[String, Long]), - componentWithImplementation + componentWithImplementation, + clock ).determineQueryParams().value.futureValue.value params should contain("c_srvcccntsrvc" -> "5") @@ -288,7 +348,7 @@ class ScenarioStatisticsTest Some(SimpleStateStatus.Running), nodesCount = 4, scenarioCategory = "Category1", - scenarioVersion = VersionId(2), + scenarioVersion = VersionId(1), createdBy = "user", fragmentsUsedCount = 2, lastDeployedAction = None, @@ -303,46 +363,64 @@ class ScenarioStatisticsTest _ => Future.successful(Right(processActivityList)), () => Future.successful(Right(componentList)), () => Future.successful(Map.empty[String, Long]), - componentWithImplementation + componentWithImplementation, + clock ).determineQueryParams().value.futureValue.value val expectedStats = Map( - AuthorsCount -> "1", - AttachmentsTotal -> "1", - AttachmentsAverage -> "1", - CategoriesCount -> "1", - CommentsTotal -> "1", - CommentsAverage -> "1", - VersionsMedian -> "2", - VersionsMax -> "2", - VersionsMin -> "2", - VersionsAverage -> "2", - UptimeInSecondsAverage -> "0", - UptimeInSecondsMax -> "0", - UptimeInSecondsMin -> "0", - ComponentsCount -> "3", - FragmentsUsedMedian -> "1", - FragmentsUsedAverage -> "1", - NodesMedian -> "3", - NodesAverage -> "2", - NodesMax -> "2", - NodesMin -> "4", - ScenarioCount -> "3", - FragmentCount -> "1", - UnboundedStreamCount -> "3", - BoundedStreamCount -> "0", - RequestResponseCount -> "1", - FlinkDMCount -> "3", - LiteK8sDMCount -> "1", - LiteEmbeddedDMCount -> "0", - UnknownDMCount -> "0", - ActiveScenarioCount -> "2", - "c_srvcccntsrvc" -> "5", - "c_cstm" -> "1", - ).map { case (k, v) => (k.toString, v) } + AuthorsCount -> 1, + AttachmentsTotal -> 1, + AttachmentsAverage -> 1, + CategoriesCount -> 1, + CommentsTotal -> 1, + CommentsAverage -> 1, + VersionsMedian -> 2, + VersionsMax -> 2, + VersionsMin -> 1, + VersionsAverage -> 1, + UptimeInSecondsAverage -> 0, + UptimeInSecondsMax -> 0, + UptimeInSecondsMin -> 0, + ComponentsCount -> 3, + FragmentsUsedMedian -> 1, + FragmentsUsedAverage -> 1, + NodesMedian -> 3, + NodesAverage -> 2, + NodesMax -> 4, + NodesMin -> 2, + ScenarioCount -> 3, + FragmentCount -> 1, + UnboundedStreamCount -> 3, + BoundedStreamCount -> 0, + RequestResponseCount -> 1, + FlinkDMCount -> 3, + LiteK8sDMCount -> 1, + LiteEmbeddedDMCount -> 0, + UnknownDMCount -> 0, + ActiveScenarioCount -> 2, + "c_srvcccntsrvc" -> 5, + "c_cstm" -> 1, + ).map { case (k, v) => (k.toString, v.toString) } params should contain allElementsOf expectedStats } + test("should provide all statistics even without any scenarios present") { + val params = new UsageStatisticsReportsSettingsService( + UsageStatisticsReportsConfig(enabled = true, Some(sampleFingerprint), None), + StatisticUrlConfig(), + mockedFingerprintService, + () => Future.successful(Right(List.empty)), + _ => Future.successful(Right(List.empty)), + () => Future.successful(Right(List.empty)), + () => Future.successful(Map.empty[String, Long]), + List.empty, + clock + ).determineQueryParams().value.futureValue.value + + params.keySet shouldBe (allScenarioRelatedStatistics ++ notScenarioRelatedStatistics).keySet + params.keySet shouldBe (emptyScenarioRelatedStatistics ++ notScenarioRelatedStatistics).keySet + } + private def processActivityList = { val scenarioActivity: ScenarioActivity = ScenarioActivity( comments = List( diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/statistics/StatisticsUrlsSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/statistics/StatisticsUrlsSpec.scala index a87d77cb865..feb2917203d 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/statistics/StatisticsUrlsSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/statistics/StatisticsUrlsSpec.scala @@ -11,6 +11,7 @@ class StatisticsUrlsSpec extends AnyFunSuite with Matchers { sut.prepare( new Fingerprint("t"), + new CorrelationId("cor_id"), Map( "q1" -> threeThousandCharsParam, "q2" -> threeThousandCharsParam, @@ -19,17 +20,18 @@ class StatisticsUrlsSpec extends AnyFunSuite with Matchers { "q5" -> threeThousandCharsParam, ) ) shouldBe List( - s"https://stats.nussknacker.io/?q1=$threeThousandCharsParam&q2=$threeThousandCharsParam&fingerprint=t", - s"https://stats.nussknacker.io/?q3=$threeThousandCharsParam&q4=$threeThousandCharsParam&fingerprint=t", - s"https://stats.nussknacker.io/?q5=$threeThousandCharsParam&fingerprint=t" + s"https://stats.nussknacker.io/?q1=$threeThousandCharsParam&q2=$threeThousandCharsParam&fingerprint=t&co_id=cor_id", + s"https://stats.nussknacker.io/?q3=$threeThousandCharsParam&q4=$threeThousandCharsParam&fingerprint=t&co_id=cor_id", + s"https://stats.nussknacker.io/?q5=$threeThousandCharsParam&fingerprint=t&co_id=cor_id" ) } test("should generate correct url with encoded params") { sut.prepare( new Fingerprint("test"), + new CorrelationId("cor_id"), Map("f" -> "a b", "v" -> "1.6.5-a&b=c") - ) shouldBe List("https://stats.nussknacker.io/?f=a+b&v=1.6.5-a%26b%3Dc&fingerprint=test") + ) shouldBe List("https://stats.nussknacker.io/?f=a+b&v=1.6.5-a%26b%3Dc&fingerprint=test&co_id=cor_id") } } diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/statistics/UsageStatisticsReportsSettingsServiceTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/statistics/UsageStatisticsReportsSettingsServiceTest.scala index 215d12f712d..cdf6f3d9f02 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/statistics/UsageStatisticsReportsSettingsServiceTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/statistics/UsageStatisticsReportsSettingsServiceTest.scala @@ -1,5 +1,7 @@ package pl.touk.nussknacker.ui.statistics +import org.mockito.invocation.InvocationOnMock +import org.mockito.stubbing.Answer import org.scalatest.EitherValues import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers @@ -8,6 +10,7 @@ import org.scalatestplus.mockito.MockitoSugar import pl.touk.nussknacker.test.PatientScalaFutures import pl.touk.nussknacker.ui.config.UsageStatisticsReportsConfig +import java.time.Clock import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future @@ -20,6 +23,15 @@ class UsageStatisticsReportsSettingsServiceTest with MockitoSugar { private val fingerprintService: FingerprintService = mock[FingerprintService] + private val sampleFingerprint = "fooFingerprint" + + private val mockedFingerprintService: FingerprintService = mock[FingerprintService]( + new Answer[Future[Either[StatisticError, Fingerprint]]] { + override def answer(invocation: InvocationOnMock): Future[Either[StatisticError, Fingerprint]] = + Future.successful(Right(new Fingerprint(sampleFingerprint))) + } + ) + test("should not generate an url if it's not configured") { val sut = new UsageStatisticsReportsSettingsService( config = UsageStatisticsReportsConfig(enabled = false, None, None), @@ -29,10 +41,65 @@ class UsageStatisticsReportsSettingsServiceTest fetchActivity = (_: List[ScenarioStatisticsInputData]) => Future.successful(Right(Nil)), fetchComponentList = () => Future.successful(Right(Nil)), fetchFeStatistics = () => Future.successful(Map.empty[String, Long]), - List.empty + components = List.empty, + designerClock = Clock.systemUTC() ) sut.prepareStatisticsUrl().futureValue shouldBe Right(Nil) } + test("should include all statisticKeys even without any scenarios created") { + val url = new UsageStatisticsReportsSettingsService( + config = UsageStatisticsReportsConfig(enabled = true, Some(sampleFingerprint), Some("source")), + urlConfig = StatisticUrlConfig(), + fingerprintService = mockedFingerprintService, + fetchNonArchivedScenariosInputData = () => Future.successful(Right(List.empty)), + fetchActivity = _ => Future.successful(Right(List.empty)), + fetchComponentList = () => Future.successful(Right(List.empty)), + fetchFeStatistics = () => Future.successful(Map.empty[String, Long]), + components = List.empty, + designerClock = Clock.systemUTC() + ).prepareStatisticsUrl().futureValue.value.reduce(_ ++ _) + + List( + AuthorsCount, + CategoriesCount, + ComponentsCount, + VersionsMedian, + AttachmentsTotal, + AttachmentsAverage, + VersionsMax, + VersionsMin, + VersionsAverage, + UptimeInSecondsAverage, + UptimeInSecondsMax, + UptimeInSecondsMin, + CommentsAverage, + CommentsTotal, + FragmentsUsedMedian, + FragmentsUsedAverage, + NodesMedian, + NodesAverage, + NodesMax, + NodesMin, + ScenarioCount, + FragmentCount, + UnboundedStreamCount, + BoundedStreamCount, + RequestResponseCount, + FlinkDMCount, + LiteK8sDMCount, + LiteEmbeddedDMCount, + UnknownDMCount, + ActiveScenarioCount, + NuSource, + NuFingerprint, + NuVersion, + CorrelationIdStat, + DesignerUptimeInSeconds, + ) + .map(_.name) + .foreach(stat => url should include(stat)) + } + }