Merge pull request #263 from Privado-Inc/dev

1. Experimental JS fixes 2. Improved results output on cloud with more metadata.
Privado-Inc · Dec 24, 2022 · fbd4df2 · fbd4df2
2 parents 5489a18 + ddba72d
commit fbd4df2
Show file tree

Hide file tree

Showing 6 changed files with 52 additions and 39 deletions.
diff --git a/src/main/scala/ai/privado/exporter/ExporterUtility.scala b/src/main/scala/ai/privado/exporter/ExporterUtility.scala
@@ -78,7 +78,8 @@ object ExporterUtility {
     if (fileName == "<empty>" || sample == "<empty>")
       None
     else {
-      val excerpt = dump(absoluteFileName, node.lineNumber)
+      val methodFullName = Traversal(node).isCall.methodFullName.headOption.getOrElse("")
+      val excerpt        = dump(absoluteFileName, node.lineNumber, methodFullName)
       Some(DataFlowSubCategoryPathExcerptModel(sample, lineNumber, columnNumber, fileName, excerpt))
     }
   }

diff --git a/src/main/scala/ai/privado/languageEngine/java/threatEngine/ThreatUtility.scala b/src/main/scala/ai/privado/languageEngine/java/threatEngine/ThreatUtility.scala
@@ -102,7 +102,7 @@ object ThreatUtility {
   ): DataFlowSubCategoryPathExcerptModel = {
 
     val lineNumber = getLineNumberOfMatchingEditText(filename, matchingTextForLine)
-    val excerpt    = Utilities.dump(filename, Some(lineNumber)) + "\n" + excerptPostfix + "\n"
+    val excerpt    = Utilities.dump(filename, Some(lineNumber), "") + "\n" + excerptPostfix + "\n"
     DataFlowSubCategoryPathExcerptModel(sample, lineNumber, -1, filename, excerpt)
   }
 

diff --git a/src/main/scala/ai/privado/languageEngine/javascript/tagger/PrivadoTagger.scala b/src/main/scala/ai/privado/languageEngine/javascript/tagger/PrivadoTagger.scala
@@ -25,7 +25,7 @@ package ai.privado.languageEngine.javascript.tagger
 
 import ai.privado.languageEngine.javascript.tagger.sink.RegularSinkTagger
 import ai.privado.languageEngine.javascript.tagger.source.IdentifierTagger
-import ai.privado.model.ConfigAndRules
+import ai.privado.model.{ConfigAndRules, NodeType}
 import ai.privado.tagger.PrivadoBaseTagger
 import ai.privado.tagger.sink.APITagger
 import ai.privado.tagger.source.LiteralTagger
@@ -42,16 +42,32 @@ class PrivadoTagger(cpg: Cpg) extends PrivadoBaseTagger {
 
   override def runTagger(rules: ConfigAndRules): Traversal[Tag] = {
 
+    val sourceRules = rules.sources
     logger.info("Starting tagging")
 
+    val literalTagger     = new LiteralTagger(cpg)
+    val identifierTagger  = new IdentifierTagger(cpg)
+    val apiTagger         = new APITagger(cpg)
+    val regularSinkTagger = new RegularSinkTagger(cpg)
     println(s"${Calendar.getInstance().getTime} - LiteralTagger invoked...")
-    new LiteralTagger(cpg).createAndApply()
+    sourceRules.foreach(rule => {
+      literalTagger.setRuleAndApply(rule)
+    })
+
     println(s"${Calendar.getInstance().getTime} - IdentifierTagger invoked...")
-    new IdentifierTagger(cpg).createAndApply()
+    sourceRules.foreach(rule => {
+      identifierTagger.setRuleAndApply(rule)
+    })
     println(s"${Calendar.getInstance().getTime} - RegularSinkTagger invoked...")
-    new RegularSinkTagger(cpg).createAndApply()
+
+    rules.sinks
+      .filter(rule => rule.nodeType.equals(NodeType.REGULAR))
+      .foreach(rule => regularSinkTagger.setRuleAndApply(rule))
     println(s"${Calendar.getInstance().getTime} - APITagger invoked...")
-    new APITagger(cpg).createAndApply()
+
+    rules.sinks
+      .filter(rule => rule.nodeType.equals(NodeType.API))
+      .foreach(rule => apiTagger.setRuleAndApply(rule))
 
     logger.info("Done with tagging")
 

diff --git a/src/main/scala/ai/privado/languageEngine/javascript/tagger/sink/RegularSinkTagger.scala b/src/main/scala/ai/privado/languageEngine/javascript/tagger/sink/RegularSinkTagger.scala
@@ -25,27 +25,21 @@ package ai.privado.languageEngine.javascript.tagger.sink
 
 import ai.privado.cache.RuleCache
 import ai.privado.model.{NodeType, RuleInfo}
+import ai.privado.tagger.PrivadoSimplePass
 import ai.privado.utility.Utilities.addRuleTags
 import io.shiftleft.codepropertygraph.generated.{Cpg, Operators}
 import io.shiftleft.passes.ConcurrentWriterCpgPass
 import io.shiftleft.semanticcpg.language._
+import overflowdb.BatchedUpdate
 
 import scala.jdk.CollectionConverters.CollectionHasAsScala
 
-class RegularSinkTagger(cpg: Cpg) extends ConcurrentWriterCpgPass[RuleInfo](cpg) {
+class RegularSinkTagger(cpg: Cpg) extends PrivadoSimplePass(cpg) {
   lazy val cacheCall = cpg.call.or(_.nameNot(Operators.ALL.asScala.toSeq: _*)).l
 
-  override def generateParts(): Array[RuleInfo] = {
-    RuleCache.getRule.sinks
-      .filter(rule => rule.nodeType.equals(NodeType.REGULAR))
-      .toArray
-  }
-
-  override def runOnPart(builder: DiffGraphBuilder, ruleInfo: RuleInfo): Unit = {
-
+  override def run(builder: BatchedUpdate.DiffGraphBuilder): Unit = {
     val sinks = cacheCall.methodFullName("(pkg.){0,1}(" + ruleInfo.combinedRulePattern + ").*").l
 
     sinks.foreach(sink => addRuleTags(builder, sink, ruleInfo))
-
   }
 }
diff --git a/src/main/scala/ai/privado/languageEngine/javascript/tagger/source/IdentifierTagger.scala b/src/main/scala/ai/privado/languageEngine/javascript/tagger/source/IdentifierTagger.scala
@@ -23,31 +23,27 @@
 
 package ai.privado.languageEngine.javascript.tagger.source
 
-import ai.privado.cache.RuleCache
-import ai.privado.model.{InternalTag, RuleInfo}
+import ai.privado.model.InternalTag
+import ai.privado.tagger.PrivadoSimplePass
 import ai.privado.utility.Utilities.{addRuleTags, storeForTag}
 import io.shiftleft.codepropertygraph.generated.Cpg
-import io.shiftleft.passes.ForkJoinParallelCpgPass
 import io.shiftleft.semanticcpg.language._
+import overflowdb.BatchedUpdate
 
-class IdentifierTagger(cpg: Cpg) extends ForkJoinParallelCpgPass[RuleInfo](cpg) {
-  override def generateParts(): Array[RuleInfo] = RuleCache.getRule.sources.toArray
-
-  override def runOnPart(builder: DiffGraphBuilder, ruleInfo: RuleInfo): Unit = {
-    RuleCache.getRule.sources.foreach(ruleInfo => {
-      val rulePattern              = ruleInfo.combinedRulePattern
-      val regexMatchingIdentifiers = cpg.identifier(rulePattern).l
-      regexMatchingIdentifiers.foreach(identifier => {
-        storeForTag(builder, identifier)(InternalTag.VARIABLE_REGEX_IDENTIFIER.toString)
-        addRuleTags(builder, identifier, ruleInfo)
-      })
+class IdentifierTagger(cpg: Cpg) extends PrivadoSimplePass(cpg) {
+  override def run(builder: BatchedUpdate.DiffGraphBuilder): Unit = {
+    val rulePattern              = ruleInfo.combinedRulePattern
+    val regexMatchingIdentifiers = cpg.identifier(rulePattern).l
+    regexMatchingIdentifiers.foreach(identifier => {
+      storeForTag(builder, identifier)(InternalTag.VARIABLE_REGEX_IDENTIFIER.toString)
+      addRuleTags(builder, identifier, ruleInfo)
+    })
 
-      val regexMatchingFieldIdentifiersIdentifiers =
-        cpg.fieldAccess.where(_.fieldIdentifier.canonicalName(rulePattern)).isCall.l
-      regexMatchingFieldIdentifiersIdentifiers.foreach(identifier => {
-        storeForTag(builder, identifier)(InternalTag.VARIABLE_REGEX_IDENTIFIER.toString)
-        addRuleTags(builder, identifier, ruleInfo)
-      })
+    val regexMatchingFieldIdentifiersIdentifiers =
+      cpg.fieldAccess.where(_.fieldIdentifier.canonicalName(rulePattern)).isCall.l
+    regexMatchingFieldIdentifiersIdentifiers.foreach(identifier => {
+      storeForTag(builder, identifier)(InternalTag.VARIABLE_REGEX_IDENTIFIER.toString)
+      addRuleTags(builder, identifier, ruleInfo)
     })
   }
 }
diff --git a/src/main/scala/ai/privado/utility/Utilities.scala b/src/main/scala/ai/privado/utility/Utilities.scala
@@ -209,8 +209,14 @@ object Utilities {
     * `lineToHighlight` is defined, then a line containing an arrow (as a source code comment) is included right before
     * that line.
     */
-  def dump(filename: String, lineToHighlight: Option[Integer]): String = {
-    val arrow: CharSequence = "/* <=== */ "
+  def dump(filename: String, lineToHighlight: Option[Integer], methodFullName: String = ""): String = {
+    val methodType = {
+      val methodInterface = methodFullName.split(":").headOption.getOrElse("")
+      if (methodInterface.contains("unresolved") || methodInterface.contains("<operator>")) ""
+      else methodInterface
+    }
+
+    val arrow: CharSequence = "/* <=== " + methodType + " */ "
     try {
       if (!filename.equals("<empty>")) {
         val lines = IOUtils.readLinesInFile(Paths.get(filename))