From 2e4628dced60698eb52bece1031e16dfe4dc99ca Mon Sep 17 00:00:00 2001 From: KhemrajSingh Rathore Date: Thu, 6 Apr 2023 12:18:35 +0530 Subject: [PATCH 1/3] =?UTF-8?q?add=20-=20soap=20api=20and=20collection=20p?= =?UTF-8?q?oint=20where=20method=20is=20denoted=20by=20webser=E2=80=A6=20(?= =?UTF-8?q?#479)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add - soap api and collection point where method is denoted by webservice annotation * refactoring and test case * more test cases * more testcase * refactoring --- .../java/processor/JavaProcessor.scala | 2 +- .../java/tagger/PrivadoTagger.scala | 22 +++-- .../tagger/Utility/SOAPTaggerUtility.scala | 41 ++++++++++ .../tagger/collection/CollectionTagger.scala | 35 ++------ .../tagger/collection/CollectionUtility.scala | 57 +++++++++---- .../collection/SOAPCollectionTagger.scala | 81 +++++++++++++++++++ .../java/tagger/sink/JavaAPITagger.scala | 15 ++-- .../java/tagger/source/IdentifierTagger.scala | 16 +++- .../ai/privado/tagger/PrivadoBaseTagger.scala | 6 +- .../collection/CollectionUtilityTest.scala | 77 ++++++++++++++++++ .../java/tagger/collection/SOAPTest.scala | 68 ++++++++++++++++ 11 files changed, 355 insertions(+), 65 deletions(-) create mode 100644 src/main/scala/ai/privado/languageEngine/java/tagger/Utility/SOAPTaggerUtility.scala create mode 100644 src/main/scala/ai/privado/languageEngine/java/tagger/collection/SOAPCollectionTagger.scala create mode 100644 src/test/scala/ai/privado/languageEngine/java/tagger/collection/CollectionUtilityTest.scala create mode 100644 src/test/scala/ai/privado/languageEngine/java/tagger/collection/SOAPTest.scala diff --git a/src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala b/src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala index 2bbf25912..bd497aeba 100644 --- a/src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala @@ -97,7 +97,7 @@ object JavaProcessor { // Run tagger println(s"${Calendar.getInstance().getTime} - Tagging source code with rules...") val taggerCache = new TaggerCache - cpg.runTagger(processedRules, taggerCache) + cpg.runTagger(processedRules, taggerCache, ScanProcessor.config) println( s"${TimeMetric.getNewTime()} - Tagging source code is done in \t\t\t- ${TimeMetric.setNewTimeToLastAndGetTimeDiff()}" ) diff --git a/src/main/scala/ai/privado/languageEngine/java/tagger/PrivadoTagger.scala b/src/main/scala/ai/privado/languageEngine/java/tagger/PrivadoTagger.scala index 228286932..3655ef630 100644 --- a/src/main/scala/ai/privado/languageEngine/java/tagger/PrivadoTagger.scala +++ b/src/main/scala/ai/privado/languageEngine/java/tagger/PrivadoTagger.scala @@ -24,13 +24,12 @@ package ai.privado.languageEngine.java.tagger import ai.privado.cache.{RuleCache, TaggerCache} -import ai.privado.entrypoint.{ScanProcessor, TimeMetric} +import ai.privado.entrypoint.{PrivadoInput, ScanProcessor, TimeMetric} import ai.privado.languageEngine.java.feeder.StorageInheritRule -import ai.privado.languageEngine.java.tagger.Utility.GRPCTaggerUtility -import ai.privado.languageEngine.java.tagger.collection.{CollectionTagger, GrpcCollectionTagger} +import ai.privado.languageEngine.java.tagger.collection.{CollectionTagger, GrpcCollectionTagger, SOAPCollectionTagger} import ai.privado.languageEngine.java.tagger.sink.{CustomInheritTagger, JavaAPITagger} import ai.privado.languageEngine.java.tagger.source.{IdentifierTagger, InSensitiveCallTagger} -import ai.privado.model.{ConfigAndRules, RuleInfo} +import ai.privado.model.ConfigAndRules import ai.privado.tagger.PrivadoBaseTagger import ai.privado.tagger.config.DBConfigTagger import ai.privado.tagger.sink.RegularSinkTagger @@ -46,7 +45,11 @@ import java.util.Calendar class PrivadoTagger(cpg: Cpg) extends PrivadoBaseTagger { private val logger = LoggerFactory.getLogger(this.getClass) - override def runTagger(rules: ConfigAndRules, taggerCache: TaggerCache): Traversal[Tag] = { + override def runTagger( + rules: ConfigAndRules, + taggerCache: TaggerCache, + privadoInputConfig: PrivadoInput + ): Traversal[Tag] = { logger.info("Starting tagging") val sourceRules = rules.sources @@ -82,7 +85,7 @@ class PrivadoTagger(cpg: Cpg) extends PrivadoBaseTagger { ) println(s"${Calendar.getInstance().getTime} - --APITagger invoked...") - new JavaAPITagger(cpg).createAndApply() + new JavaAPITagger(cpg, privadoInputConfig).createAndApply() println( s"${TimeMetric.getNewTime()} - --APITagger is done in \t\t\t\t- ${TimeMetric.setNewTimeToStageLastAndGetTimeDiff()}" ) @@ -98,12 +101,17 @@ class PrivadoTagger(cpg: Cpg) extends PrivadoBaseTagger { } println(s"${Calendar.getInstance().getTime} - --CollectionTagger invoked...") - val collectionTagger = new CollectionTagger(cpg, sourceRules) new CollectionTagger(cpg, RuleCache.getRule.sources).createAndApply() println( s"${TimeMetric.getNewTime()} - --CollectionTagger is done in \t\t\t- ${TimeMetric.setNewTimeToStageLastAndGetTimeDiff()}" ) + println(s"${Calendar.getInstance().getTime} - --SOAPCollectionTagger invoked...") + new SOAPCollectionTagger(cpg, sourceRules).createAndApply() + println( + s"${TimeMetric.getNewTime()} - --SOAPCollectionTagger is done in \t\t\t- ${TimeMetric.setNewTimeToStageLastAndGetTimeDiff()}" + ) + println(s"${Calendar.getInstance().getTime} - --GrpcCollectionTagger invoked...") new GrpcCollectionTagger(cpg, sourceRules).createAndApply() println( diff --git a/src/main/scala/ai/privado/languageEngine/java/tagger/Utility/SOAPTaggerUtility.scala b/src/main/scala/ai/privado/languageEngine/java/tagger/Utility/SOAPTaggerUtility.scala new file mode 100644 index 000000000..1273c32db --- /dev/null +++ b/src/main/scala/ai/privado/languageEngine/java/tagger/Utility/SOAPTaggerUtility.scala @@ -0,0 +1,41 @@ +/* + * This file is part of Privado OSS. + * + * Privado is an open source static code analysis tool to discover data flows in the code. + * Copyright (C) 2022 Privado, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * For more information, contact support@privado.ai + * + */ + +package ai.privado.languageEngine.java.tagger.Utility + +import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.codepropertygraph.generated.nodes.{Call, Method} +import io.shiftleft.semanticcpg.language._ + +object SOAPTaggerUtility { + + def getAPIMethods(cpg: Cpg): List[Method] = { + cpg.annotation.name("WebService").typeDecl.method.l + } + + def getAPICallNodes(cpg: Cpg): List[Call] = { + implicit val resolver: ICallResolver = NoResolve + getAPIMethods(cpg).callIn.l + } + +} diff --git a/src/main/scala/ai/privado/languageEngine/java/tagger/collection/CollectionTagger.scala b/src/main/scala/ai/privado/languageEngine/java/tagger/collection/CollectionTagger.scala index 9392ca5d8..884862692 100644 --- a/src/main/scala/ai/privado/languageEngine/java/tagger/collection/CollectionTagger.scala +++ b/src/main/scala/ai/privado/languageEngine/java/tagger/collection/CollectionTagger.scala @@ -24,19 +24,13 @@ package ai.privado.languageEngine.java.tagger.collection import ai.privado.cache.RuleCache -import ai.privado.model.{Constants, InternalTag, RuleInfo} -import ai.privado.tagger.PrivadoSimplePass -import ai.privado.utility.Utilities._ +import ai.privado.model.RuleInfo import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.{Annotation, Method} import io.shiftleft.passes.ForkJoinParallelCpgPass import io.shiftleft.semanticcpg.language._ import org.slf4j.LoggerFactory -import overflowdb.BatchedUpdate -import overflowdb.traversal.Traversal import scala.collection.mutable -import scala.util.{Failure, Success, Try} class CollectionTagger(cpg: Cpg, sourceRuleInfos: List[RuleInfo]) extends ForkJoinParallelCpgPass[RuleInfo](cpg) { private val logger = LoggerFactory.getLogger(this.getClass) @@ -54,13 +48,16 @@ class CollectionTagger(cpg: Cpg, sourceRuleInfos: List[RuleInfo]) extends ForkJo .name(combinedRulePatterns) .filter(_.typeDecl.nonEmpty) .foreach(classAnnotation => { - classUrlMap.addOne(classAnnotation.typeDecl.head.id() -> getCollectionUrl(classAnnotation)) + classUrlMap + .addOne(classAnnotation.typeDecl.head.id() -> CollectionUtility.getUrlFromAnnotation(classAnnotation)) }) val collectionMethodsCache = cpg.annotation .name(combinedRulePatterns) .filter(_.method.nonEmpty) .map(matchedAnnotation => { - methodUrlMap.addOne(matchedAnnotation.method.head.id() -> getCollectionUrl(matchedAnnotation)) + methodUrlMap.addOne( + matchedAnnotation.method.head.id() -> CollectionUtility.getUrlFromAnnotation(matchedAnnotation) + ) matchedAnnotation }) .method @@ -84,24 +81,4 @@ class CollectionTagger(cpg: Cpg, sourceRuleInfos: List[RuleInfo]) extends ForkJo ) } - /** Returns rest Url for this annotation - * @param parameterIn - * @return - */ - private def getCollectionUrl(annotation: Annotation) = { - Try(annotation.parameterAssign.order(1).astChildren.order(2).l.head) match { - case Success(url) => url.code - case Failure(_) => - Try(annotation.parameterAssign.order(1).head) match { - case Success(url) => url.code - case Failure(_) => - Try(annotation) match { - case Success(url) => url.code - case Failure(e) => - logger.debug("Exception : ", e) - "" - } - } - } - } } diff --git a/src/main/scala/ai/privado/languageEngine/java/tagger/collection/CollectionUtility.scala b/src/main/scala/ai/privado/languageEngine/java/tagger/collection/CollectionUtility.scala index 659498f01..efe575b98 100644 --- a/src/main/scala/ai/privado/languageEngine/java/tagger/collection/CollectionUtility.scala +++ b/src/main/scala/ai/privado/languageEngine/java/tagger/collection/CollectionUtility.scala @@ -3,15 +3,18 @@ package ai.privado.languageEngine.java.tagger.collection import ai.privado.model.{Constants, InternalTag, RuleInfo} import ai.privado.utility.Utilities._ import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.Method +import io.shiftleft.codepropertygraph.generated.nodes.{Annotation, Method} import io.shiftleft.semanticcpg.language._ +import org.slf4j.LoggerFactory import overflowdb.BatchedUpdate.DiffGraphBuilder import overflowdb.traversal.Traversal -import scala.util.{Failure, Success, Try} +import scala.util.{Failure, Success, Try} import scala.collection.mutable object CollectionUtility { + + private val logger = LoggerFactory.getLogger(this.getClass) def tagDirectSources( builder: DiffGraphBuilder, collectionMethods: List[Method], @@ -48,26 +51,22 @@ object CollectionUtility { // Implementation to also mark the collection points which use derived type declaration as there parameters val derivedTypeDecl = (getAllDerivedTypeDecl(cpg, InternalTag.OBJECT_OF_SENSITIVE_CLASS_BY_MEMBER_NAME.toString) ++ getAllDerivedTypeDecl(cpg, InternalTag.OBJECT_OF_SENSITIVE_CLASS_BY_MEMBER_TYPE.toString) ++ - getAllDerivedTypeDecl(cpg, InternalTag.OBJECT_OF_SENSITIVE_CLASS_BY_INHERITANCE.toString)).distinct + getAllDerivedTypeDecl(cpg, InternalTag.OBJECT_OF_SENSITIVE_CLASS_BY_INHERITANCE.toString)).toSet - val collectionPointsFromDerivedTypeDecl = Traversal(collectionMethods).flatMap(collectionMethod => { + val collectionPointsFromDerivedTypeDecl = collectionMethods.flatMap(collectionMethod => { val parameters = collectionMethod.parameter.where(_.typeFullName.filter(fullName => derivedTypeDecl.contains(fullName))) if (parameters.isEmpty) { None } else { - parameters.foreach(parameter => { - val derivedReferencingIdentifier = parameter.referencingIdentifiers - .whereNot(_.code("this")) - .where(_.tag.name(Constants.privadoDerived + ".*")) - .l - if (derivedReferencingIdentifier.nonEmpty) { - Try(derivedReferencingIdentifier.head.tag.name(Constants.privadoDerived + ".*")) match { - case Success(refIdentifierTags) => - refIdentifierTags.foreach(refTag => storeForTag(builder, parameter)(refTag.name, refTag.value)) - } - } - }) + // Have removed the earlier code, where we were fetching all the referencing identifiers of parameter and then tagging, because we were missing on cases where the parameter is not used in the code + parameters + .whereNot(_.code("this")) + .foreach(parameter => { + parameter.tag + .name(Constants.privadoDerived + ".*") + .foreach(refTag => storeForTag(builder, parameter)(refTag.name, refTag.value)) + }) collectionMethod } }) @@ -120,4 +119,30 @@ object CollectionUtility { } } } + + /** This function gets the URL from an Annotation object. It will first look for the URL in the parameterAssign node + * order 1, then if that fails it will look for the URL in the parameterAssign node order 2, then if that fails it + * will look for the URL in the typeDecl node, and finally if that fails it will look for the URL in the method node. + * If none of these succeed, it will return an empty string. + * @param parameterIn + * @return + */ + def getUrlFromAnnotation(annotation: Annotation): String = { + annotation.parameterAssign.order(1).astChildren.order(2).l.headOption match { + case Some(url) => url.code + case None => + annotation.parameterAssign.order(1).headOption match { + case Some(url) => url.code + case None => + annotation.typeDecl.headOption match { + case Some(typeDeclNode) => typeDeclNode.name + case None => + annotation.method.headOption match { + case Some(methodNode) => methodNode.name + case None => "" + } + } + } + } + } } diff --git a/src/main/scala/ai/privado/languageEngine/java/tagger/collection/SOAPCollectionTagger.scala b/src/main/scala/ai/privado/languageEngine/java/tagger/collection/SOAPCollectionTagger.scala new file mode 100644 index 000000000..973aee051 --- /dev/null +++ b/src/main/scala/ai/privado/languageEngine/java/tagger/collection/SOAPCollectionTagger.scala @@ -0,0 +1,81 @@ +/* + * This file is part of Privado OSS. + * + * Privado is an open source static code analysis tool to discover data flows in the code. + * Copyright (C) 2022 Privado, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * For more information, contact support@privado.ai + * + */ + +package ai.privado.languageEngine.java.tagger.collection + +import ai.privado.languageEngine.java.tagger.Utility.SOAPTaggerUtility +import ai.privado.model.{CatLevelOne, Language, NodeType, RuleInfo} +import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.passes.CpgPass +import org.slf4j.LoggerFactory +import io.shiftleft.semanticcpg.language._ + +import scala.collection.immutable.HashMap +import scala.collection.mutable + +class SOAPCollectionTagger(cpg: Cpg, sourceRuleInfos: List[RuleInfo]) extends CpgPass(cpg) { + private val logger = LoggerFactory.getLogger(this.getClass) + + override def run(builder: DiffGraphBuilder): Unit = { + logger.info("Tagging collection points for SOAP") + + val soapCollectionMethods = SOAPTaggerUtility.getAPIMethods(cpg) + + // Create a hardcoded rule specially for SOAP so we adhere to CollectionExporter style of operation + val ruleInfo = RuleInfo( + "Collections.Annotation.SOAP", + "SOAP Annotation", + "", + Array[String](), + List[String]("WebService"), + false, + "", + HashMap[String, String](), + NodeType.REGULAR, + "", + CatLevelOne.COLLECTIONS, + "SOAP", + Language.JAVA, + Array[String]() + ) + + val classUrlMap = mutable.HashMap[Long, String]() + + cpg.annotation + .name(ruleInfo.combinedRulePattern) + .filter(_.typeDecl.nonEmpty) + .foreach(classAnnotation => { + classUrlMap + .addOne(classAnnotation.typeDecl.head.id() -> ("/" + CollectionUtility.getUrlFromAnnotation(classAnnotation))) + }) + + CollectionUtility.tagDirectSources( + builder, + soapCollectionMethods, + sourceRuleInfos, + ruleInfo, + classUrlMap = classUrlMap + ) + CollectionUtility.tagDerivedSources(cpg, builder, soapCollectionMethods, ruleInfo, classUrlMap = classUrlMap) + } +} diff --git a/src/main/scala/ai/privado/languageEngine/java/tagger/sink/JavaAPITagger.scala b/src/main/scala/ai/privado/languageEngine/java/tagger/sink/JavaAPITagger.scala index b6ab356c4..cdb80e01b 100644 --- a/src/main/scala/ai/privado/languageEngine/java/tagger/sink/JavaAPITagger.scala +++ b/src/main/scala/ai/privado/languageEngine/java/tagger/sink/JavaAPITagger.scala @@ -23,9 +23,9 @@ package ai.privado.languageEngine.java.tagger.sink import ai.privado.cache.{AppCache, RuleCache} -import ai.privado.entrypoint.ScanProcessor +import ai.privado.entrypoint.{PrivadoInput, ScanProcessor} import ai.privado.languageEngine.java.language._ -import ai.privado.languageEngine.java.tagger.Utility.GRPCTaggerUtility +import ai.privado.languageEngine.java.tagger.Utility.{GRPCTaggerUtility, SOAPTaggerUtility} import ai.privado.metric.MetricHandler import ai.privado.model.{Constants, Language, NodeType, RuleInfo} import ai.privado.tagger.utility.APITaggerUtility.sinkTagger @@ -53,7 +53,7 @@ object APITaggerVersionJava extends Enumeration { val SkipTagger, V1Tagger, V2Tagger = Value } -class JavaAPITagger(cpg: Cpg) extends ForkJoinParallelCpgPass[RuleInfo](cpg) { +class JavaAPITagger(cpg: Cpg, privadoInputConfig: PrivadoInput) extends ForkJoinParallelCpgPass[RuleInfo](cpg) { private val logger = LoggerFactory.getLogger(this.getClass) val cacheCall: List[Call] = cpg.call.where(_.nameNot("( { @@ -156,7 +158,11 @@ class IdentifierTagger(cpg: Cpg, taggerCache: TaggerCache) extends ForkJoinParal val typeDeclVal = typeDeclTuple._2 val typeDeclMember = typeDeclTuple._1 taggerCache.addItemToTypeDeclMemberCache(typeDeclVal, ruleInfo.id, typeDeclMember) - val impactedObjects = cpg.identifier.where(_.typeFullName(typeDeclVal)).whereNot(_.code("this")) + val impactedObjects = + cpg.identifier.where(_.typeFullName(typeDeclVal)).whereNot(_.code("this")).l ::: cpg.parameter + .where(_.typeFullName(typeDeclVal)) + .whereNot(_.code("this")) + .l impactedObjects.foreach(impactedObject => { if (impactedObject.tag.nameExact(Constants.id).l.isEmpty) { storeForTag(builder, impactedObject)(InternalTag.OBJECT_OF_SENSITIVE_CLASS_BY_MEMBER_TYPE.toString) @@ -215,7 +221,11 @@ class IdentifierTagger(cpg: Cpg, taggerCache: TaggerCache) extends ForkJoinParal .typeDeclExtendingTypeDeclCache(typeDeclVal) .addOne(ruleInfo.id -> cpg.typeDecl.where(_.fullNameExact(typeDeclName)).head) - val impactedObjects = cpg.identifier.where(_.typeFullName(typeDeclVal)).whereNot(_.code("this")) + val impactedObjects = + cpg.identifier.where(_.typeFullName(typeDeclVal)).whereNot(_.code("this")).l ::: cpg.parameter + .where(_.typeFullName(typeDeclVal)) + .whereNot(_.code("this")) + .l impactedObjects.foreach(impactedObject => { if (impactedObject.tag.nameExact(Constants.id).l.isEmpty) { storeForTag(builder, impactedObject)(InternalTag.OBJECT_OF_SENSITIVE_CLASS_BY_INHERITANCE.toString) diff --git a/src/main/scala/ai/privado/tagger/PrivadoBaseTagger.scala b/src/main/scala/ai/privado/tagger/PrivadoBaseTagger.scala index a9fd50778..7cb9d76ca 100644 --- a/src/main/scala/ai/privado/tagger/PrivadoBaseTagger.scala +++ b/src/main/scala/ai/privado/tagger/PrivadoBaseTagger.scala @@ -1,13 +1,15 @@ package ai.privado.tagger import ai.privado.cache.TaggerCache +import ai.privado.entrypoint.PrivadoInput import ai.privado.model.ConfigAndRules import io.shiftleft.codepropertygraph.generated.nodes.Tag import overflowdb.traversal.Traversal abstract class PrivadoBaseTagger { - def runTagger(rules: ConfigAndRules): Traversal[Tag] = ??? - def runTagger(rules: ConfigAndRules, taggerCache: TaggerCache): Traversal[Tag] = ??? + def runTagger(rules: ConfigAndRules): Traversal[Tag] = ??? + def runTagger(rules: ConfigAndRules, taggerCache: TaggerCache): Traversal[Tag] = ??? + def runTagger(rules: ConfigAndRules, taggerCache: TaggerCache, privadoInputConfig: PrivadoInput): Traversal[Tag] = ??? } diff --git a/src/test/scala/ai/privado/languageEngine/java/tagger/collection/CollectionUtilityTest.scala b/src/test/scala/ai/privado/languageEngine/java/tagger/collection/CollectionUtilityTest.scala new file mode 100644 index 000000000..a82c7b352 --- /dev/null +++ b/src/test/scala/ai/privado/languageEngine/java/tagger/collection/CollectionUtilityTest.scala @@ -0,0 +1,77 @@ +/* + * This file is part of Privado OSS. + * + * Privado is an open source static code analysis tool to discover data flows in the code. + * Copyright (C) 2022 Privado, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * For more information, contact support@privado.ai + * + */ + +package ai.privado.languageEngine.java.tagger.collection + +import ai.privado.languageEngine.java.JavaTaggingTestBase +import io.shiftleft.semanticcpg.language._ + +class CollectionUtilityTest extends JavaTaggingTestBase { + + override def beforeAll(): Unit = { + super.beforeAll() + + } + + override val javaFileContents: String = + """ + | + |@RequestMapping("/api/public/user") + |public class AuthenticationService { + | + | + | @PostMapping("/signup") + | public UserProfileD signup(@RequestBody String firstName) { + | } + | + | @PostMapping(value = "/signin") + | public UserProfileD signin(@RequestBody String firstName) { + | } + | + | @PostMapping + | public UserProfileD sample3(@RequestBody String firstName) { + | } + | + |} + | + | + |""".stripMargin + + "Get Url for annotation" should { + "give url for sample1" in { + CollectionUtility.getUrlFromAnnotation(cpg.typeDecl.annotation.head) shouldBe "/api/public/user" + } + } + + "Get Url for annotation" should { + "give url for sample2" in { + CollectionUtility.getUrlFromAnnotation(cpg.method("signup").annotation.head) shouldBe "/signup" + } + } + + "Get Url for annotation" should { + "give url for sample3" in { + CollectionUtility.getUrlFromAnnotation(cpg.method("sample3").annotation.head) shouldBe "sample3" + } + } +} diff --git a/src/test/scala/ai/privado/languageEngine/java/tagger/collection/SOAPTest.scala b/src/test/scala/ai/privado/languageEngine/java/tagger/collection/SOAPTest.scala new file mode 100644 index 000000000..ea8d0ef65 --- /dev/null +++ b/src/test/scala/ai/privado/languageEngine/java/tagger/collection/SOAPTest.scala @@ -0,0 +1,68 @@ +/* + * This file is part of Privado OSS. + * + * Privado is an open source static code analysis tool to discover data flows in the code. + * Copyright (C) 2022 Privado, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * For more information, contact support@privado.ai + * + */ + +package ai.privado.languageEngine.java.tagger.collection + +import ai.privado.languageEngine.java.JavaTaggingTestBase +import ai.privado.languageEngine.java.tagger.source.IdentifierTagger +import ai.privado.model.Constants +import io.shiftleft.semanticcpg.language._ + +class SOAPTest extends JavaTaggingTestBase { + + override def beforeAll(): Unit = { + super.beforeAll() + new IdentifierTagger(cpg, taggerCache).createAndApply() + new SOAPCollectionTagger(cpg, sourceRule).createAndApply() + + } + + override val javaFileContents: String = + """ + |@WebService(name = "ContourSoapService", targetNamespace = "http://v3.ws.contour.jamasoftware.com/") + |@XmlSeeAlso({ + | ObjectFactory.class + |}) + |public interface ContourSoapService { + | + | + | @WebMethod(action = "updateUser") + | @WebResult(targetNamespace = "") + | @RequestWrapper(localName = "updateUser", targetNamespace = "http://v3.ws.contour.jamasoftware.com/", className = "com.jamasoftware.examples.soap.wsdl.UpdateUser") + | @ResponseWrapper(localName = "updateUserResponse", targetNamespace = "http://v3.ws.contour.jamasoftware.com/", className = "com.jamasoftware.examples.soap.wsdl.UpdateUserResponse") + | WsUser updateUser( + | @WebParam(name = "token", targetNamespace = "") + | WsAuth token, + | @WebParam(name = "firstName", targetNamespace = "") + | String firstName); + |} + |""".stripMargin + + "SOAP collection point" should { + "be tagged" in { + cpg.method + .where(_.tag.nameExact(Constants.id).valueExact("Collections.Annotation.SOAP")) + .size shouldBe 1 + } + } +} From 910421b57bb122228466b5a1b7a9d852269eed80 Mon Sep 17 00:00:00 2001 From: Ankit Kumar <118803988+ankit-privado@users.noreply.github.com> Date: Thu, 6 Apr 2023 16:06:37 +0530 Subject: [PATCH 2/3] Added Additional column in audit discovery report (#482) * Added Additional column in audit discovery sheet * code refactoring * Handled multiple endpoint url and added background color * Refactoring code * Added filter on collection and Refactoring code --- .../privado/audit/AuditReportConstants.scala | 42 ++++ .../privado/audit/DataElementDiscovery.scala | 183 +++++++++++++++--- .../ai/privado/exporter/ExcelExporter.scala | 31 ++- .../java/audit/DataElementDiscoveryTest.scala | 46 ++++- .../audit/TestData/AuditTestClassData.scala | 14 ++ 5 files changed, 274 insertions(+), 42 deletions(-) create mode 100644 src/main/scala/ai/privado/audit/AuditReportConstants.scala diff --git a/src/main/scala/ai/privado/audit/AuditReportConstants.scala b/src/main/scala/ai/privado/audit/AuditReportConstants.scala new file mode 100644 index 000000000..6b6e02efc --- /dev/null +++ b/src/main/scala/ai/privado/audit/AuditReportConstants.scala @@ -0,0 +1,42 @@ +package ai.privado.audit + +object AuditReportConstants { + + val AUDIT_ELEMENT_DISCOVERY_SHEET_NAME = "Element-Discovery" + + val AUDIT_EMPTY_CELL_VALUE = "--" + + val AUDIT_CHECKED_VALUE = "YES" + + val AUDIT_NOT_CHECKED_VALUE = "NO" + + val AUDIT_BUILT_IN_CLASS_REGEX = + "\\b((int|byte|short|long|float|double|boolean|char)|((java\\.|javax\\.|org\\.|com\\.sun\\.|com\\.oracle\\.)([A-Z][a-zA-Z0-9_]*(\\.[A-Z][a-zA-Z0-9_]*)*)))\\b" + + val ELEMENT_DISCOVERY_CLASS_NAME = "Class" + + val ELEMENT_DISCOVERY_FILE_NAME = "File Name" + + val ELEMENT_DISCOVERY_MEMBER_NAME = "Member" + + val ELEMENT_DISCOVERY_MEMBER_TYPE = "Member Type" + + val ELEMENT_DISCOVERY_TAGGED_NAME = "Tagged" + + val ELEMENT_DISCOVERY_SOURCE_RULE_ID = "Source Rule ID" + + val ELEMENT_DISCOVERY_INPUT_COLLECTION = "Input to Collection" + + val ELEMENT_DISCOVERY_COLLECTION_ENDPOINT = "Collection Endpoint Path" + + val ELEMENT_DISCOVERY_METHOD_NAME = "Collection Method Full Name" + + val ELEMENT_DISCOVERY_EXCLUDE_CLASS_NAME_REGEX = "^(.*)(Controller|Service|Impl|Helper|Util|Processor|Dao)$" + + val ELEMENT_DISCOVERY_GET_SET_METHOD_REGEX = "^(get|set).*" + + val ELEMENT_DISCOVERY_OVERRIDE_METHOD_REGEX = "^(hascode|equals)" + + val ELEMENT_DISCOVERY_GETTER_SETTER_REGEX = ".*(Getter|Setter).*" + +} diff --git a/src/main/scala/ai/privado/audit/DataElementDiscovery.scala b/src/main/scala/ai/privado/audit/DataElementDiscovery.scala index d4dcd8356..c3071715b 100644 --- a/src/main/scala/ai/privado/audit/DataElementDiscovery.scala +++ b/src/main/scala/ai/privado/audit/DataElementDiscovery.scala @@ -1,8 +1,9 @@ package ai.privado.audit import ai.privado.cache.TaggerCache +import ai.privado.model.{CatLevelOne, Constants, InternalTag} import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.{Member, MethodParameterIn} +import io.shiftleft.codepropertygraph.generated.nodes.{Member, TypeDecl} import io.shiftleft.semanticcpg.language._ import org.slf4j.LoggerFactory @@ -14,8 +15,6 @@ object DataElementDiscovery { private val logger = LoggerFactory.getLogger(getClass) - private val excludeClassNameRegex = "^(.*)(Controller|Service|Impl|Helper|Util|Processor)$" - // Get list of Class Name having getter and setter method def getSourceUsingRules(xtocpg: Try[Cpg]): List[String] = { logger.info("Process Class Name from cpg") @@ -25,11 +24,11 @@ object DataElementDiscovery { // Get DTO/Entity Class name val typeDeclList = cpg.typeDecl .filter(_.order > 0) - .whereNot(_.name(excludeClassNameRegex)) + .whereNot(_.name(AuditReportConstants.ELEMENT_DISCOVERY_EXCLUDE_CLASS_NAME_REGEX)) .or( - _.where(_.method.name("^(get|set).*")), - _.where(_.method.name("^(hascode|equals)")), - _.where(_.annotation.name(".*(Getter|Setter).*")) + _.where(_.method.name(AuditReportConstants.ELEMENT_DISCOVERY_GET_SET_METHOD_REGEX)), + _.where(_.method.name(AuditReportConstants.ELEMENT_DISCOVERY_OVERRIDE_METHOD_REGEX)), + _.where(_.annotation.name(AuditReportConstants.ELEMENT_DISCOVERY_GETTER_SETTER_REGEX)) ) .toList typeDeclList.foreach(node => { @@ -68,7 +67,7 @@ object DataElementDiscovery { val typeDeclList = cpg.typeDecl .filter(_.order > 0) .where(_.fullName(pattern)) - .whereNot(_.name(excludeClassNameRegex)) + .whereNot(_.name(AuditReportConstants.ELEMENT_DISCOVERY_EXCLUDE_CLASS_NAME_REGEX)) .toList typeDeclList.foreach(typeDecl => derivedClassName += typeDecl.fullName) }) @@ -83,16 +82,16 @@ object DataElementDiscovery { } // Get list of member variable present in given class - def getMemberUsingClassName(xtocpg: Try[Cpg], classNameSet: Set[String]): mutable.Map[String, List[Member]] = { + def getMemberUsingClassName(xtocpg: Try[Cpg], classNameSet: Set[String]): Map[TypeDecl, List[Member]] = { logger.info("Process Member Name from cpg") - val memberInfoMap = mutable.Map[String, List[Member]]() + val memberInfoMap = mutable.HashMap[TypeDecl, List[Member]]() xtocpg match { case Success(cpg) => { classNameSet.foreach(className => { - // Get member variable of class and put it into map - val memberInfo = cpg.typeDecl.filter(_.order > 0).where(_.fullName(className)).member.toList - memberInfoMap.put(className, memberInfo) + cpg.typeDecl + .where(_.fullName(className)) + .foreach(typeDeclNode => memberInfoMap.put(typeDeclNode, typeDeclNode.member.l)) }) } case Failure(exception) => { @@ -102,7 +101,7 @@ object DataElementDiscovery { } } logger.info("Successfully Processed member name from cpg") - memberInfoMap + memberInfoMap.toMap } // Get Collection Input Class Name @@ -112,7 +111,11 @@ object DataElementDiscovery { xtocpg match { case Success(cpg) => { // Get tagged collection input list - val parameterList = cpg.parameter.where(_.tag).l + val parameterList = cpg.parameter + .where(_.method.tag.nameExact(Constants.catLevelOne).valueExact(CatLevelOne.COLLECTIONS.name)) + .whereNot(_.typeFullName(AuditReportConstants.AUDIT_BUILT_IN_CLASS_REGEX)) + .whereNot(_.typeFullName(AuditReportConstants.ELEMENT_DISCOVERY_EXCLUDE_CLASS_NAME_REGEX)) + .l parameterList.foreach(parameter => { collectionInputList += parameter.typeFullName }) @@ -126,11 +129,45 @@ object DataElementDiscovery { collectionInputList.toList } + def getCollectionMethodInfo(xtocpg: Try[Cpg]): Map[String, ListBuffer[CollectionMethodInfo]] = { + // className -> (MethodName, Endpoint) + val collectionMethodInfoMap = new mutable.HashMap[String, ListBuffer[CollectionMethodInfo]]() + xtocpg match { + case Success(cpg) => { + val parameterList = cpg.parameter + .where(_.method.tag.nameExact(Constants.catLevelOne).valueExact(CatLevelOne.COLLECTIONS.name)) + .whereNot(_.typeFullName(AuditReportConstants.AUDIT_BUILT_IN_CLASS_REGEX)) + .whereNot(_.typeFullName(AuditReportConstants.ELEMENT_DISCOVERY_EXCLUDE_CLASS_NAME_REGEX)) + .l + parameterList.foreach(parameter => { + + // Get complete Endpoint path + val endpointTag = parameter.method.tag.where(_.name(InternalTag.COLLECTION_METHOD_ENDPOINT.toString)).head + // Append endpoint and method name when Method parameter having same type + if (!collectionMethodInfoMap.contains(parameter.typeFullName)) { + collectionMethodInfoMap + .put(parameter.typeFullName, new ListBuffer[CollectionMethodInfo]) + } + + val collectionMethodInfo = collectionMethodInfoMap(parameter.typeFullName) + collectionMethodInfo += CollectionMethodInfo(parameter.method.code, endpointTag.value) + }) + } + case Failure(exception) => { + println("Failed to process collection method info from cpg") + logger.debug("Failed to process collection method info from cpg", exception) + println(exception.printStackTrace()) + } + } + collectionMethodInfoMap.toMap + } + def processDataElementDiscovery(xtocpg: Try[Cpg], taggerCache: TaggerCache): List[List[String]] = { logger.info("Initiated the audit engine") - val classNameRuleList = getSourceUsingRules(xtocpg) - val collectionInputList = getCollectionInputList(xtocpg) - val derivedClassName = extractClassFromPackage(xtocpg, (classNameRuleList ++ collectionInputList).toSet) + val classNameRuleList = getSourceUsingRules(xtocpg) + val collectionInputList = getCollectionInputList(xtocpg) + val collectionMethodInfo = getCollectionMethodInfo(xtocpg) + val derivedClassName = extractClassFromPackage(xtocpg, (classNameRuleList ++ collectionInputList).toSet) val memberInfo = getMemberUsingClassName(xtocpg, (classNameRuleList ++ collectionInputList ++ derivedClassName).toSet) val workbookResult = new ListBuffer[List[String]]() @@ -151,33 +188,119 @@ object DataElementDiscovery { } // Header List - workbookResult += List("Class", "Member", "Tagged", "Source Rule ID", "Input to Collection") + workbookResult += List( + AuditReportConstants.ELEMENT_DISCOVERY_CLASS_NAME, + AuditReportConstants.ELEMENT_DISCOVERY_FILE_NAME, + AuditReportConstants.ELEMENT_DISCOVERY_MEMBER_NAME, + AuditReportConstants.ELEMENT_DISCOVERY_MEMBER_TYPE, + AuditReportConstants.ELEMENT_DISCOVERY_TAGGED_NAME, + AuditReportConstants.ELEMENT_DISCOVERY_SOURCE_RULE_ID, + AuditReportConstants.ELEMENT_DISCOVERY_INPUT_COLLECTION, + AuditReportConstants.ELEMENT_DISCOVERY_COLLECTION_ENDPOINT, + AuditReportConstants.ELEMENT_DISCOVERY_METHOD_NAME + ) // Construct the excel sheet and fill the data try { memberInfo.foreach { case (key, value) => { - val isCollectionInput = if (collectionInputList.contains(key)) "YES" else "NO" - if (taggedMemberInfo.contains(key)) { - workbookResult += List(key, "--", "YES", "", isCollectionInput) - val ruleMemberInfo = taggedMemberInfo.getOrElse(key, new mutable.HashMap[String, String]) + val isCollectionInput = if (collectionInputList.contains(key.fullName)) "YES" else "NO" + if (taggedMemberInfo.contains(key.fullName)) { + if (collectionMethodInfo.contains(key.fullName)) { + collectionMethodInfo(key.fullName).foreach(info => { + workbookResult += List( + key.fullName, + key.file.head.name, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + AuditReportConstants.AUDIT_CHECKED_VALUE, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + isCollectionInput, + info.endpoint, + info.methodDetail + ) + }) + } else { + workbookResult += List( + key.fullName, + key.file.head.name, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + AuditReportConstants.AUDIT_CHECKED_VALUE, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + isCollectionInput, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE + ) + } + val ruleMemberInfo = taggedMemberInfo.getOrElse(key.fullName, new mutable.HashMap[String, String]) value.foreach(member => { if (ruleMemberInfo.contains(member.name)) { workbookResult += List( - "", + key.fullName, + key.file.head.name, member.name, - "YES", + member.typeFullName, + AuditReportConstants.AUDIT_CHECKED_VALUE, ruleMemberInfo.getOrElse(member.name, "Default value"), - "" + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE ) } else { - workbookResult += List("", member.name, "NO", "--", "") + workbookResult += List( + key.fullName, + key.file.head.name, + member.name, + member.typeFullName, + AuditReportConstants.AUDIT_NOT_CHECKED_VALUE, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE + ) } }) } else { - workbookResult += List(key, "--", "NO", "", isCollectionInput) + if (collectionMethodInfo.contains(key.fullName)) { + collectionMethodInfo(key.fullName).foreach(info => { + workbookResult += List( + key.fullName, + key.file.head.name, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + AuditReportConstants.AUDIT_NOT_CHECKED_VALUE, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + isCollectionInput, + info.endpoint, + info.methodDetail + ) + }) + } else { + workbookResult += List( + key.fullName, + key.file.head.name, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + AuditReportConstants.AUDIT_NOT_CHECKED_VALUE, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + isCollectionInput, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE + ) + } value.foreach(member => { - workbookResult += List("", member.name, "NO", "--", "") + workbookResult += List( + key.fullName, + key.file.head.name, + member.name, + member.typeFullName, + AuditReportConstants.AUDIT_NOT_CHECKED_VALUE, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE, + AuditReportConstants.AUDIT_EMPTY_CELL_VALUE + ) }) } } @@ -190,4 +313,6 @@ object DataElementDiscovery { } workbookResult.toList } + + case class CollectionMethodInfo(var methodDetail: String, var endpoint: String) } diff --git a/src/main/scala/ai/privado/exporter/ExcelExporter.scala b/src/main/scala/ai/privado/exporter/ExcelExporter.scala index 87ee4e9eb..9819ac83a 100644 --- a/src/main/scala/ai/privado/exporter/ExcelExporter.scala +++ b/src/main/scala/ai/privado/exporter/ExcelExporter.scala @@ -1,9 +1,10 @@ package ai.privado.exporter +import ai.privado.audit.AuditReportConstants import ai.privado.model.Constants.outputDirectoryName import better.files.File -import org.apache.poi.ss.usermodel.{Cell, Row, Sheet, Workbook} -import org.apache.poi.xssf.usermodel.XSSFWorkbook +import org.apache.poi.ss.usermodel.{Cell, FillPatternType, IndexedColors, Row, Sheet, Workbook} +import org.apache.poi.xssf.usermodel.{XSSFCellStyle, XSSFColor, XSSFWorkbook} import org.slf4j.LoggerFactory import java.io.FileOutputStream @@ -17,7 +18,7 @@ object ExcelExporter { logger.info("Initiated the Audit exporter engine") try { val workbook: Workbook = new XSSFWorkbook() - val sheet: Sheet = workbook.createSheet("Member-Tag") + val sheet: Sheet = workbook.createSheet(AuditReportConstants.AUDIT_ELEMENT_DISCOVERY_SHEET_NAME) // Iterate over the data and write it to the sheet for ((rowValues, rowIndex) <- output.zipWithIndex) { @@ -27,6 +28,9 @@ object ExcelExporter { cell.setCellValue(cellValue) } } + + // Changed Background colour when tagged + changeTaggedBackgroundColour(workbook, List(4, 6)) logger.info("Successfully added audit report to excel file") // create directory if not exist @@ -46,4 +50,25 @@ object ExcelExporter { Left(ex.toString) } } + + private def changeTaggedBackgroundColour(workbook: Workbook, columnList: List[Integer]) = { + + val sheet = workbook.getSheet(AuditReportConstants.AUDIT_ELEMENT_DISCOVERY_SHEET_NAME) + + val greenCellStyle: XSSFCellStyle = workbook.createCellStyle().asInstanceOf[XSSFCellStyle] + val greenColor: XSSFColor = new XSSFColor(IndexedColors.LIGHT_GREEN, null) + greenCellStyle.setFillForegroundColor(greenColor) + greenCellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND) + + val rowIterator = sheet.rowIterator() + while (rowIterator.hasNext) { + val row = rowIterator.next() + columnList.foreach(columnNo => { + val cell = row.getCell(columnNo) + if (cell != null && cell.getStringCellValue == AuditReportConstants.AUDIT_CHECKED_VALUE) { + cell.setCellStyle(greenCellStyle) + } + }) + } + } } diff --git a/src/test/scala/ai/privado/languageEngine/java/audit/DataElementDiscoveryTest.scala b/src/test/scala/ai/privado/languageEngine/java/audit/DataElementDiscoveryTest.scala index 994e46c5f..cefcdb996 100644 --- a/src/test/scala/ai/privado/languageEngine/java/audit/DataElementDiscoveryTest.scala +++ b/src/test/scala/ai/privado/languageEngine/java/audit/DataElementDiscoveryTest.scala @@ -3,9 +3,9 @@ import ai.privado.audit.DataElementDiscovery import ai.privado.languageEngine.java.audit.TestData.AuditTestClassData import ai.privado.languageEngine.java.tagger.collection.CollectionTagger import ai.privado.languageEngine.java.tagger.source.IdentifierTagger +import io.shiftleft.codepropertygraph.generated.nodes.Member import scala.collection.mutable -import scala.collection.mutable.ListBuffer import scala.util.Try class DataElementDiscoveryTest extends DataElementDiscoveryTestBase { @@ -28,6 +28,7 @@ class DataElementDiscoveryTest extends DataElementDiscoveryTestBase { testClassMap.put("Salary", AuditTestClassData.salaryLombok) testClassMap.put("AddressController", AuditTestClassData.addressController) testClassMap.put("Invoice", AuditTestClassData.invoice) + testClassMap.put("AdminDao", AuditTestClassData.adminDao) testClassMap.toMap } @@ -43,6 +44,7 @@ class DataElementDiscoveryTest extends DataElementDiscoveryTestBase { classNameList should not contain ("com.test.privado.Controller.AddressController") classNameList should not contain ("com.test.privado.Controller.UserController") classNameList should not contain ("com.test.privado.Entity.Address") + classNameList should not contain ("com.test.privado.Dao.AdminDao") } "Test discovery of class Name in package from class name" in { @@ -57,6 +59,7 @@ class DataElementDiscoveryTest extends DataElementDiscoveryTestBase { discoveryList should contain("com.test.privado.Entity.Address") discoveryList should not contain ("com.test.privado.Controller.AddressController") discoveryList should not contain ("com.test.privado.Controller.UserController") + discoveryList should not contain ("com.test.privado.Dao.AdminDao") } "Test class member variable" in { @@ -64,10 +67,19 @@ class DataElementDiscoveryTest extends DataElementDiscoveryTestBase { val memberMap = DataElementDiscovery.getMemberUsingClassName(Try(cpg), classList.toSet) - memberMap(classList.head).size shouldBe 1 - memberMap(classList.head).head.name shouldBe ("firstName") - memberMap(classList(1)).size shouldBe 1 - memberMap(classList(1)).head.name shouldBe ("accountNo") + val classMemberMap = new mutable.HashMap[String, List[Member]]() + + memberMap.foreach { case (key, value) => + classMemberMap.put(key.fullName, value) + } + + classMemberMap.keys.toList should contain("com.test.privado.Entity.User") + classMemberMap("com.test.privado.Entity.User").size shouldBe 1 + classMemberMap("com.test.privado.Entity.User").head.name should equal("firstName") + + classMemberMap.keys.toList should contain("com.test.privado.Entity.Account") + classMemberMap("com.test.privado.Entity.Account").size shouldBe 1 + classMemberMap("com.test.privado.Entity.Account").head.name should equal("accountNo") } "Test Collection discovery" in { @@ -79,21 +91,26 @@ class DataElementDiscoveryTest extends DataElementDiscoveryTestBase { collectionList should not contain ("com.test.privado.Entity.Address") collectionList should not contain ("com.test.privado.Controller.AddressController") collectionList should not contain ("com.test.privado.Controller.UserController") + collectionList should not contain ("com.test.privado.Dao.AdminDao") } "Test final discovery result" in { - val classNameList = new ListBuffer[String]() - val memberList = new ListBuffer[String]() + val classNameList = new mutable.HashSet[String]() + val memberList = new mutable.HashSet[String]() val sourceRuleIdMap = new mutable.HashMap[String, String]() val collectionTagMap = new mutable.HashMap[String, String]() + val endpointMap = new mutable.HashMap[String, String]() + val methodNameMap = new mutable.HashMap[String, String]() val workbookList = DataElementDiscovery.processDataElementDiscovery(Try(cpg), taggerCache) workbookList.foreach(row => { classNameList += row.head - memberList += row(1) - sourceRuleIdMap.put(row(1), row(3)) - collectionTagMap.put(row.head, row(4)) + memberList += row(2) + sourceRuleIdMap.put(row(2), row(5)) + if (!collectionTagMap.contains(row.head)) collectionTagMap.put(row.head, row(6)) + if (!endpointMap.contains(row.head)) endpointMap.put(row.head, row(7)) + if (!methodNameMap.contains(row.head)) methodNameMap.put(row.head, row(8)) }) // Validate class name in result @@ -104,6 +121,7 @@ class DataElementDiscoveryTest extends DataElementDiscoveryTestBase { classNameList should contain("com.test.privado.Entity.Address") classNameList should not contain ("com.test.privado.Controller.UserController") classNameList should not contain ("com.test.privado.Controller.AddressController") + classNameList should not contain ("com.test.privado.Dao.AdminDao") // Validate class member in result memberList should contain("houseNo") @@ -118,6 +136,14 @@ class DataElementDiscoveryTest extends DataElementDiscoveryTestBase { // validate collection Tag in result collectionTagMap("com.test.privado.Entity.User").toString should equal("YES") + + // validate collection endpoint in result + endpointMap("com.test.privado.Entity.User").toString should equal("/user/add") + + // validate collection method name in result + methodNameMap("com.test.privado.Entity.User").toString should equal( + "public String userHandler(@RequestBody User user)" + ) } } } diff --git a/src/test/scala/ai/privado/languageEngine/java/audit/TestData/AuditTestClassData.scala b/src/test/scala/ai/privado/languageEngine/java/audit/TestData/AuditTestClassData.scala index 727db62c9..ac7a62ee3 100644 --- a/src/test/scala/ai/privado/languageEngine/java/audit/TestData/AuditTestClassData.scala +++ b/src/test/scala/ai/privado/languageEngine/java/audit/TestData/AuditTestClassData.scala @@ -106,4 +106,18 @@ object AuditTestClassData { | } |} |""".stripMargin + + val adminDao = + """ + |package com.test.privado.Dao; + | + |public class AdminDao { + | + | public String adminName; + | + | public String getAdminName() { + | return adminName; + | } + |} + |""".stripMargin } From 37544d3cb475b6d291bf2711c3e80969c46272f4 Mon Sep 17 00:00:00 2001 From: KhemrajSingh Rathore Date: Thu, 6 Apr 2023 18:08:11 +0530 Subject: [PATCH 3/3] trigger (#489) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cee43bbda..020a1f0b8 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Privado Core ============================================= -Branch structure +Branch structure main - This branch will contain the released version of the code.