diff --git a/build.sbt b/build.sbt index 4e353991f6c..5884e32a71d 100644 --- a/build.sbt +++ b/build.sbt @@ -1126,7 +1126,7 @@ lazy val defaultHelpers = (project in utils("default-helpers")) .settings( name := "nussknacker-default-helpers" ) - .dependsOn(mathUtils, testUtils % Test, scenarioCompiler % "test->test;test->compile") + .dependsOn(mathUtils, commonUtils, testUtils % Test, scenarioCompiler % "test->test;test->compile") lazy val testUtils = (project in utils("test-utils")) .settings(commonSettings) diff --git a/defaultModel/src/main/scala/pl/touk/nussknacker/defaultmodel/DefaultConfigCreator.scala b/defaultModel/src/main/scala/pl/touk/nussknacker/defaultmodel/DefaultConfigCreator.scala index bb2c0ea08e3..dee4b9e2344 100644 --- a/defaultModel/src/main/scala/pl/touk/nussknacker/defaultmodel/DefaultConfigCreator.scala +++ b/defaultModel/src/main/scala/pl/touk/nussknacker/defaultmodel/DefaultConfigCreator.scala @@ -18,6 +18,7 @@ class DefaultConfigCreator extends EmptyProcessConfigCreator { "DATE_FORMAT" -> anyCategory(dateFormat), "UTIL" -> anyCategory(util), "RANDOM" -> anyCategory(random), + "BASE64" -> anyCategory(base64) ), List() ) diff --git a/docs/Changelog.md b/docs/Changelog.md index 7ca1455bed4..7537057424a 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -47,9 +47,11 @@ * toBigIntegerOrNull * toBigDecimal * toBigDecimalOrNull +* [#6995](https://github.com/TouK/nussknacker/pull/6995) Add `toJson` and `toJsonString` conversions (in the `#CONV` helper) +* [#6995](https://github.com/TouK/nussknacker/pull/6995) Add `#BASE64` helper to decode/encode Base64 values * [#6826](https://github.com/TouK/nussknacker/pull/6826) Security fix: added validation of expression used inside indexer for Maps and Lists (for example `{1,2,3}[#otherList.remove(1) == null ? 0 : 0]`). This allowed executing - some types of unallowed expressions. + some types of not allowed expressions. * [#6880](https://github.com/TouK/nussknacker/pull/6880) Performance optimization of generating Avro messages with unions - shorter message in logs * [#6766](https://github.com/TouK/nussknacker/pull/6766) Scenario labels support - you can assign labels to scenarios and use them to filter the scenario list diff --git a/docs/scenarios_authoring/Spel.md b/docs/scenarios_authoring/Spel.md index a144437c135..d404727d2e2 100644 --- a/docs/scenarios_authoring/Spel.md +++ b/docs/scenarios_authoring/Spel.md @@ -289,17 +289,17 @@ Explicit conversions are available in utility classes and build-in java conversi Nussknacker comes with the following helpers: -| Helper | Functions | -|---------------|------------------------------------------------| -| `COLLECTION` | Operations on collections | -| `CONV` | General conversion functions | -| `DATE` | Date operations (conversions, useful helpers) | -| `DATE_FORMAT` | Date formatting/parsing operations | -| `GEO` | Simple distance measurements | -| `NUMERIC` | Number parsing | -| `RANDOM` | Random value generators | -| `UTIL` | Various utilities (e.g. identifier generation) | - +| Helper | Functions | +|---------------|--------------------------------------------------------------------| +| `COLLECTION` | Operations on collections | +| `CONV` | General conversion functions | +| `DATE` | Date operations (conversions, useful helpers) | +| `DATE_FORMAT` | Date formatting/parsing operations | +| `GEO` | Simple distance measurements | +| `NUMERIC` | Number parsing | +| `RANDOM` | Random value generators | +| `UTIL` | Various utilities (e.g. identifier generation) | +| `BASE64` | Encoding & decoding [Base64](https://en.wikipedia.org/wiki/Base64) | ## Handling date/time. diff --git a/engine/flink/test-utils/src/main/scala/pl/touk/nussknacker/engine/flink/test/ClassDiscoveryBaseTest.scala b/engine/flink/test-utils/src/main/scala/pl/touk/nussknacker/engine/flink/test/ClassDiscoveryBaseTest.scala index 1ba594e71df..0ff385fa406 100644 --- a/engine/flink/test-utils/src/main/scala/pl/touk/nussknacker/engine/flink/test/ClassDiscoveryBaseTest.scala +++ b/engine/flink/test-utils/src/main/scala/pl/touk/nussknacker/engine/flink/test/ClassDiscoveryBaseTest.scala @@ -2,6 +2,7 @@ package pl.touk.nussknacker.engine.flink.test import cats.data.NonEmptyList import cats.implicits.toFunctorOps +import com.typesafe.scalalogging.LazyLogging import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} import io.circe.parser.parse import io.circe.syntax.EncoderOps @@ -13,7 +14,7 @@ import org.scalatest.matchers.should.Matchers import org.springframework.util.ClassUtils import pl.touk.nussknacker.engine.ModelData import pl.touk.nussknacker.engine.api.generics.{MethodTypeInfo, Parameter} -import pl.touk.nussknacker.engine.api.typed.typing.{TypedClass, TypingResult, Unknown} +import pl.touk.nussknacker.engine.api.typed.typing.{TypedClass, TypingResult} import pl.touk.nussknacker.engine.api.typed.{TypeEncoders, TypingResultDecoder} import pl.touk.nussknacker.engine.definition.clazz.{ ClassDefinition, @@ -30,7 +31,7 @@ import pl.touk.nussknacker.engine.api.CirceUtil._ import scala.util.Properties -trait ClassDiscoveryBaseTest extends AnyFunSuite with Matchers with Inside { +trait ClassDiscoveryBaseTest extends AnyFunSuite with Matchers with Inside with LazyLogging { protected def model: ModelData @@ -68,6 +69,7 @@ trait ClassDiscoveryBaseTest extends AnyFunSuite with Matchers with Inside { val types = model.modelDefinitionWithClasses.classDefinitions.all if (Option(System.getenv("CLASS_EXTRACTION_PRINT")).exists(_.toBoolean)) { val fileName = s"${Properties.tmpDir}/${getClass.getSimpleName}-result.json" + logger.info(s"CLASS_EXTRACTION_PRINT is set. The file JSON file will be stored in '$fileName'") FileUtils.write(new File(fileName), encode(types), StandardCharsets.UTF_8) } val parsed = parse(ResourceLoader.load(outputResource)).toOption.get diff --git a/engine/flink/tests/src/test/resources/extractedTypes/defaultModel.json b/engine/flink/tests/src/test/resources/extractedTypes/defaultModel.json index f24de2fc4eb..f614ba1f8c9 100644 --- a/engine/flink/tests/src/test/resources/extractedTypes/defaultModel.json +++ b/engine/flink/tests/src/test/resources/extractedTypes/defaultModel.json @@ -13124,6 +13124,60 @@ ] } }, + { + "clazzName": {"refClazzName": "pl.touk.nussknacker.engine.util.functions.base64$"}, + "methods": { + "decode": [ + { + "description": "Decode Base64 value to String", + "name": "decode", + "signature": { + "noVarArgs": [ + {"name": "value", "refClazz": {"refClazzName": "java.lang.String"}} + ], + "result": {"refClazzName": "java.lang.String"} + } + } + ], + "encode": [ + { + "description": "Encode String value to Base64", + "name": "encode", + "signature": { + "noVarArgs": [ + {"name": "value", "refClazz": {"refClazzName": "java.lang.String"}} + ], + "result": {"refClazzName": "java.lang.String"} + } + } + ], + "urlSafeDecode": [ + { + "description": "Decode URL-safe Base64 value to String", + "name": "urlSafeDecode", + "signature": { + "noVarArgs": [ + {"name": "value", "refClazz": {"refClazzName": "java.lang.String"}} + ], + "result": {"refClazzName": "java.lang.String"} + } + } + ], + "urlSafeEncode": [ + { + "description": "Encode String value to URL-safe Base64", + "name": "urlSafeEncode", + "signature": { + "noVarArgs": [ + {"name": "value", "refClazz": {"refClazzName": "java.lang.String"}} + ], + "result": {"refClazzName": "java.lang.String"} + } + } + ] + }, + "staticMethods": {} + }, { "clazzName": {"refClazzName": "pl.touk.nussknacker.engine.util.functions.collection$"}, "methods": { @@ -14110,6 +14164,42 @@ } } ], + "toJson": [ + { + "description": "Convert String value to JSON", + "name": "toJson", + "signature": { + "noVarArgs": [ + {"name": "value", "refClazz": {"refClazzName": "java.lang.String"}} + ], + "result": {"type": "Unknown"} + } + } + ], + "toJsonOrNull": [ + { + "description": "Convert String value to JSON or null in case of failure", + "name": "toJsonOrNull", + "signature": { + "noVarArgs": [ + {"name": "value", "refClazz": {"refClazzName": "java.lang.String"}} + ], + "result": {"type": "Unknown"} + } + } + ], + "toJsonString": [ + { + "description": "Convert JSON to String", + "name": "toJsonString", + "signature": { + "noVarArgs": [ + {"name": "value", "refClazz": {"type": "Unknown"}} + ], + "result": {"refClazzName": "java.lang.String"} + } + } + ], "toLong": [ { "description": "Convert any value to Long or throw exception in case of failure", @@ -15521,4 +15611,4 @@ ] } } -] +] \ No newline at end of file diff --git a/utils/default-helpers/src/main/scala/pl/touk/nussknacker/engine/util/functions/base64.scala b/utils/default-helpers/src/main/scala/pl/touk/nussknacker/engine/util/functions/base64.scala new file mode 100644 index 00000000000..3f44b9c574c --- /dev/null +++ b/utils/default-helpers/src/main/scala/pl/touk/nussknacker/engine/util/functions/base64.scala @@ -0,0 +1,31 @@ +package pl.touk.nussknacker.engine.util.functions + +import pl.touk.nussknacker.engine.api.{Documentation, HideToString, ParamName} + +import java.util.Base64 + +object base64 extends Base64Utils + +trait Base64Utils extends HideToString { + + @Documentation(description = "Decode Base64 value to String") + def decode(@ParamName("value") value: String): String = { + new String(Base64.getDecoder.decode(value.getBytes("UTF-8"))) + } + + @Documentation(description = "Encode String value to Base64") + def encode(@ParamName("value") value: String): String = { + new String(Base64.getEncoder.encode(value.getBytes("UTF-8"))) + } + + @Documentation(description = "Decode URL-safe Base64 value to String") + def urlSafeDecode(@ParamName("value") value: String): String = { + new String(Base64.getUrlDecoder.decode(value.getBytes("UTF-8"))) + } + + @Documentation(description = "Encode String value to URL-safe Base64") + def urlSafeEncode(@ParamName("value") value: String): String = { + new String(Base64.getUrlEncoder.withoutPadding().encode(value.getBytes("UTF-8"))) + } + +} diff --git a/utils/default-helpers/src/main/scala/pl/touk/nussknacker/engine/util/functions/conversion.scala b/utils/default-helpers/src/main/scala/pl/touk/nussknacker/engine/util/functions/conversion.scala index 1d5e55762b0..52350def8a0 100644 --- a/utils/default-helpers/src/main/scala/pl/touk/nussknacker/engine/util/functions/conversion.scala +++ b/utils/default-helpers/src/main/scala/pl/touk/nussknacker/engine/util/functions/conversion.scala @@ -4,6 +4,7 @@ import pl.touk.nussknacker.engine.api.generics.GenericType import pl.touk.nussknacker.engine.api.{Documentation, HideToString, ParamName} import pl.touk.nussknacker.engine.util.functions.ConversionUtils.{stringToBigInteger, stringToBoolean} import pl.touk.nussknacker.engine.util.functions.NumericUtils.ToNumberTypingFunction +import pl.touk.nussknacker.engine.util.json.{JsonUtils, ToJsonEncoder} import scala.util.Try @@ -138,6 +139,30 @@ trait ConversionUtils extends HideToString { case _ => null } + @Documentation(description = "Convert String value to JSON") + def toJson(@ParamName("value") value: String): Any = { + toJsonEither(value).toTry.get + } + + @Documentation(description = "Convert String value to JSON or null in case of failure") + def toJsonOrNull(@ParamName("value") value: String): Any = { + toJsonEither(value).getOrElse(null) + } + + @Documentation(description = "Convert JSON to String") + def toJsonString(@ParamName("value") value: Any): String = { + jsonEncoder.encode(value).noSpaces + } + + private def toJsonEither(value: String): Either[Throwable, Any] = { + io.circe.parser.parse(value) match { + case Right(json) => Right(JsonUtils.jsonToAny(json)) + case Left(ex) => Left(new IllegalArgumentException(s"Cannot convert [$value] to JSON", ex)) + } + } + + private lazy val jsonEncoder = new ToJsonEncoder(true, this.getClass.getClassLoader) + } object ConversionUtils { diff --git a/utils/default-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/Base64UtilsSpec.scala b/utils/default-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/Base64UtilsSpec.scala new file mode 100644 index 00000000000..5e8bda10938 --- /dev/null +++ b/utils/default-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/Base64UtilsSpec.scala @@ -0,0 +1,28 @@ +package pl.touk.nussknacker.engine.util.functions + +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +class Base64UtilsSpec extends AnyFunSuite with BaseSpelSpec with Matchers { + + test("encode") { + evaluate[String]("#BASE64.encode('{\"foo\": 1}')") shouldBe "eyJmb28iOiAxfQ==" + evaluate[String]("#BASE64.encode('')") shouldBe "" + } + + test("decode") { + evaluate[String]("#BASE64.decode('eyJmb28iOiAxfQ==')") shouldBe """{"foo": 1}""" + evaluate[String]("#BASE64.decode('')") shouldBe "" + } + + test("urlSafeEncode") { + evaluate[String]("#BASE64.urlSafeEncode('{\"foo\": 1}')") shouldBe "eyJmb28iOiAxfQ" + evaluate[String]("#BASE64.urlSafeEncode('')") shouldBe "" + } + + test("urlSafeDecode") { + evaluate[String]("#BASE64.urlSafeDecode('eyJmb28iOiAxfQ')") shouldBe """{"foo": 1}""" + evaluate[String]("#BASE64.urlSafeDecode('')") shouldBe "" + } + +} diff --git a/utils/default-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/BaseSpelSpec.scala b/utils/default-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/BaseSpelSpec.scala index c6fcafb4659..8f23f0c655e 100644 --- a/utils/default-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/BaseSpelSpec.scala +++ b/utils/default-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/BaseSpelSpec.scala @@ -27,7 +27,8 @@ trait BaseSpelSpec { "DATE_FORMAT" -> new DateFormatUtils(Locale.US), "UTIL" -> util, "NUMERIC" -> numeric, - "CONV" -> conversion + "CONV" -> conversion, + "BASE64" -> base64 ) private val parser = SpelExpressionParser.default( diff --git a/utils/default-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/ConversionUtilsSpec.scala b/utils/default-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/ConversionUtilsSpec.scala index ab3aba78e53..54ab7ac6263 100644 --- a/utils/default-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/ConversionUtilsSpec.scala +++ b/utils/default-helpers/src/test/scala/pl/touk/nussknacker/engine/util/functions/ConversionUtilsSpec.scala @@ -5,6 +5,9 @@ import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import org.scalatest.prop.TableDrivenPropertyChecks._ import pl.touk.nussknacker.engine.spel.SpelExpressionEvaluationException +import java.math.{BigDecimal => JBigDecimal} +import java.util.{List => JList} +import java.util.{Map => JMap} class ConversionUtilsSpec extends AnyFunSuite with BaseSpelSpec with Matchers { @@ -143,4 +146,84 @@ class ConversionUtilsSpec extends AnyFunSuite with BaseSpelSpec with Matchers { } } + test("parse JSON") { + Table( + ("expression", "expected"), + ("#CONV.toJson('null')", null), + ("#CONV.toJson('\"str\"')", "str"), + ("#CONV.toJson('1')", JBigDecimal.valueOf(1)), + ("#CONV.toJson('true')", true), + ("#CONV.toJson('[]')", JList.of()), + ("#CONV.toJson('[{}]')", JList.of[JMap[Nothing, Nothing]](JMap.of())), + ("#CONV.toJson('[1, \"str\", true]')", JList.of(JBigDecimal.valueOf(1), "str", true)), + ("#CONV.toJson('{}')", JMap.of()), + ( + "#CONV.toJson('{ \"a\": 1, \"b\": true, \"c\": \"str\", \"d\": [], \"e\": {} }')", + JMap.of( + "a", + JBigDecimal.valueOf(1), + "b", + true, + "c", + "str", + "d", + JList.of(), + "e", + JMap.of() + ) + ) + ).forEvery { (expression, expected) => + evaluateAny(expression) shouldBe expected + } + + Table( + ("expression", "expected"), + ("#CONV.toJson('{ \"a\": 1, \"b\": true, \"c\": \"str\", \"d\": [], \"e\": {} }')", "Unknown") + ).forEvery { (expression, expected) => + evaluateType(expression, types = Map.empty) shouldBe expected.valid + } + } + + test("fail to parse JSON when invalid string is passed") { + val caught = intercept[SpelExpressionEvaluationException] { + evaluateAny("""#CONV.toJson('{ "a": 1 ')""") + } + caught.getMessage should include("""Cannot convert [{ "a": 1 ] to JSON""") + } + + test("return null when parsing to JSON is not possible") { + evaluateAny("""#CONV.toJsonOrNull('{ "a": 1 ')""") should be(null) + } + + test("to stringified JSON") { + Table( + ("expression", "expected"), + ("""#CONV.toJsonString(null)""", "null"), + ("""#CONV.toJsonString('str')""", "\"str\""), + ("""#CONV.toJsonString(1)""", "1"), + ("""#CONV.toJsonString(true)""", "true"), + ("""#CONV.toJsonString({})""", """[]"""), + ("""#CONV.toJsonString({ 1, "str", true })""", """[1,"str",true]"""), + ( + """#CONV.toJsonString({ a: 1, b: true, c: "str", d: {}, e: { f: 2 } })""", + """{"e":{"f":2},"a":1,"b":true,"c":"str","d":[]}""" + ), + ).forEvery { (expression, expected) => + evaluateAny(expression) shouldBe expected + } + + Table( + ("expression", "expected"), + ("""#CONV.toJsonString(null)""", "String"), + ("""#CONV.toJsonString('str')""", "String"), + ("""#CONV.toJsonString(1)""", "String"), + ("""#CONV.toJsonString(true)""", "String"), + ("""#CONV.toJsonString({})""", "String"), + ("""#CONV.toJsonString({ 1, "str", true })""", "String"), + ("""#CONV.toJsonString({ a: 1, b: true, c: "str", d: {}, e: { f: 2 } })""", "String"), + ).forEvery { (expression, expected) => + evaluateType(expression, types = Map.empty) shouldBe expected.valid + } + } + }