Skip to content

Commit

Permalink
Merge pull request #369 from Privado-Inc/dev
Browse files Browse the repository at this point in the history 
Release: 24th Feb 2023: Rule segregation
  • Loading branch information
@pandurangpatil
pandurangpatil authored Mar 1, 2023
2 parents 6b786d3 + 23d2c65 commit 46605a9
Show file tree
Hide file tree
Showing 11 changed files with 98 additions and 51 deletions.
10 changes: 7 additions & 3 deletions build.sbt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ ThisBuild / version := sys.env.getOrElse("BUILD_VERSION", "dev-SNAPSHOT")
// parsed by project/Versions.scala, updated by updateDependencies.sh

val cpgVersion = "1.3.593"
val joernVersion = "1.1.1455"
val joernVersion = "1.1.1461"
val overflowdbVersion = "1.167"

//External dependency versions
Expand Down Expand Up @@ -62,8 +62,8 @@ ThisBuild / resolvers ++= Seq(
"Gradle Releases" at "https://repo.gradle.org/gradle/libs-releases",
Resolver.sonatypeRepo("snapshots")
)
lazy val astGenDlUrl = "https://github.com/max-leuthaeuser/astgen/releases/download/latest/"
lazy val astGenBinaryNames = Seq("astgen-linux", "astgen-macos", "astgen-win.exe")
lazy val astGenDlUrl = "https://github.com/joernio/astgen/releases/download/v2.14.0/"
lazy val astGenBinaryNames = Seq("astgen-linux", "astgen-macos", "astgen-win.exe", "astgen-macos-arm")

lazy val astGenDlTask = taskKey[Unit](s"Download astgen binaries")
astGenDlTask := {
Expand Down Expand Up @@ -106,3 +106,7 @@ resolvers += "privado--core" at "https://" + sys.env.get("CODE_ARTIFACT_URL").ge
ThisBuild / publishTo := Some(
"privado--core" at "https://" + sys.env.get("CODE_ARTIFACT_URL").getOrElse("") + "/maven/core"
)

lazy val root = (project in file("."))
.enablePlugins(BuildInfoPlugin)
.settings(buildInfoKeys := Seq[BuildInfoKey]("joernVersion" -> joernVersion), buildInfoPackage := "privado_core")
2 changes: 1 addition & 1 deletion buildspec.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ phases:
LATEST_TAG="$(git describe --tags --abbrev=0)"
VERSION=${LATEST_TAG:1}
else
LATEST_TAG=$(curl -i "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" https://api.github.com/repos/Privado-Inc/privado-core/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
LATEST_TAG=$(curl -i https://api.github.com/repos/Privado-Inc/privado-core/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
VERSION_INITIAL=${LATEST_TAG:1}
VERSION=$VERSION_INITIAL-$CODEBUILD_BUILD_NUMBER
fi
Expand Down
1 change: 1 addition & 0 deletions project/plugins.sbt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,5 @@ addSbtPlugin("com.github.sbt" % "sbt-findbugs" % "2.0.0")
addSbtPlugin("com.dwijnand" % "sbt-dynver" % "4.1.1")
addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.7")
addSbtPlugin("io.shiftleft" % "sbt-overflowdb" % "2.26")
addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.11.0")
//addSbtPlugin("org.scala-sbt" % "sbt-maven-resolver" % "0.1.0")
2 changes: 2 additions & 0 deletions src/main/scala/ai/privado/entrypoint/ScanProcessor.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ import java.util.Calendar
import scala.collection.parallel.CollectionConverters.ImmutableIterableIsParallelizable
import scala.sys.exit
import scala.util.{Failure, Success, Try}
import privado_core.BuildInfo

object ScanProcessor extends CommandProcessor {
private val logger = LoggerFactory.getLogger(this.getClass)
Expand Down Expand Up @@ -280,6 +281,7 @@ object ScanProcessor extends CommandProcessor {
override def process(): Either[String, Unit] = {
println(s"Privado CLI Version: ${Environment.privadoVersionCli.getOrElse(Constants.notDetected)}")
println(s"Privado Core Version: ${Environment.privadoVersionCore}")
println(s"Privado Language Engine Version: ${BuildInfo.joernVersion}")
if (!File(config.sourceLocation.head).isWritable) {
println(s"Warning: Privado doesn't have write permission on give repo location - ${config.sourceLocation.head}")
}
Expand Down
18 changes: 11 additions & 7 deletions src/main/scala/ai/privado/exporter/JSONExporter.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ import ai.privado.model.exporter.ViolationEncoderDecoder._
import ai.privado.model.exporter.CollectionEncoderDecoder._
import ai.privado.model.exporter.SinkEncoderDecoder._
import better.files.File
import privado_core.BuildInfo
import io.circe.Json
import io.circe.syntax.EncoderOps
import io.joern.dataflowengineoss.language.Path
Expand Down Expand Up @@ -83,13 +84,16 @@ object JSONExporter {
val policyAndThreatExporter = new PolicyAndThreatExporter(cpg, dataflows)
val output = mutable.LinkedHashMap[String, Json]()
try {
output.addOne(Constants.coreVersion -> Environment.privadoVersionCore.asJson)
output.addOne(Constants.cliVersion -> Environment.privadoVersionCli.getOrElse(Constants.notDetected).asJson)
output.addOne(Constants.mainVersion -> AppCache.privadoVersionMain.asJson)
output.addOne(Constants.createdAt -> Calendar.getInstance().getTimeInMillis.asJson)
output.addOne(Constants.repoName -> AppCache.repoName.asJson)
output.addOne(Constants.gitMetadata -> GitMetaDataExporter.getMetaData(repoPath).asJson)
output.addOne(Constants.localScanPath -> AppCache.localScanPath.asJson)

output.addOne(Constants.coreVersion -> Environment.privadoVersionCore.asJson)
output.addOne(Constants.cliVersion -> Environment.privadoVersionCli.getOrElse(Constants.notDetected).asJson)
output.addOne(Constants.mainVersion -> AppCache.privadoVersionMain.asJson)
output.addOne(Constants.privadoLanguageEngineVersion -> BuildInfo.joernVersion.asJson)
output.addOne(Constants.createdAt -> Calendar.getInstance().getTimeInMillis.asJson)
output.addOne(Constants.repoName -> AppCache.repoName.asJson)
output.addOne(Constants.gitMetadata -> GitMetaDataExporter.getMetaData(repoPath).asJson)
output.addOne(Constants.localScanPath -> AppCache.localScanPath.asJson)

val sources = Future(
sourceExporter.getSources
) // Future creates a thread and starts resolving the function call asynchronously
Expand Down
11 changes: 6 additions & 5 deletions src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,12 @@ object JavaProcessor {
s"${TimeMetric.getNewTime()} - Run oss data flow is done in \t\t\t- ${TimeMetric.setNewTimeToLastAndGetTimeDiff()}"
)

// Unresolved function report
if (config.showUnresolvedFunctionsReport) {
val path = s"${config.sourceLocation.head}/${Constants.outputDirectoryName}"
UnresolvedReportUtility.reportUnresolvedMethods(xtocpg, path, Language.JAVA)
}

// Run tagger
println(s"${Calendar.getInstance().getTime} - Tagging source code with rules...")
cpg.runTagger(processedRules)
Expand Down Expand Up @@ -183,11 +189,6 @@ object JavaProcessor {
cpg
}

if (config.showUnresolvedFunctionsReport) {
val path = s"${config.sourceLocation.head}/${Constants.outputDirectoryName}"
UnresolvedReportUtility.reportUnresolvedMethods(xtocpg, path, Language.JAVA)
}

val msg = processCPG(xtocpg, processedRules, sourceRepoLocation)

// Delete the delomboked directory after scanning is completed
Expand Down
2 changes: 1 addition & 1 deletion ...ain/scala/ai/privado/languageEngine/javascript/passes/methodfullname/MethodFullName.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ class MethodFullName(cpg: Cpg) extends ForkJoinParallelCpgPass[(String, String,
(
staticImport.importedAs.getOrElse(""),
staticImport.importedEntity.getOrElse(""),
staticImport.file.name.head,
staticImport.file.name.headOption.getOrElse(""),
"pkg."
)
)
Expand Down
10 changes: 6 additions & 4 deletions src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptProcessor.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,12 @@ object JavascriptProcessor {
new MethodFullName(cpg).createAndApply()
new MethodFullNameFromIdentifier(cpg).createAndApply()
new MethodFullNameForEmptyNodes(cpg).createAndApply()

// Unresolved function report
if (config.showUnresolvedFunctionsReport) {
val path = s"${config.sourceLocation.head}/${Constants.outputDirectoryName}"
UnresolvedReportUtility.reportUnresolvedMethods(xtocpg, path, Language.JAVASCRIPT)
}
logger.info("=====================")

// Run tagger
Expand Down Expand Up @@ -126,10 +132,6 @@ object JavascriptProcessor {
val cpgconfig =
Config(inputPath = absoluteSourceLocation)
val xtocpg = new JsSrc2Cpg().createCpgWithAllOverlays(cpgconfig)
if (config.showUnresolvedFunctionsReport) {
val path = s"${config.sourceLocation.head}/${Constants.outputDirectoryName}"
UnresolvedReportUtility.reportUnresolvedMethods(xtocpg, path, Language.JAVASCRIPT)
}
processCPG(xtocpg, processedRules, sourceRepoLocation)
}

Expand Down
10 changes: 6 additions & 4 deletions src/main/scala/ai/privado/languageEngine/python/processor/PythonProcessor.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,12 @@ object PythonProcessor {
// Apply OSS Dataflow overlay
new OssDataFlow(new OssDataFlowOptions()).run(new LayerCreatorContext(cpg))

// Unresolved function report
if (config.showUnresolvedFunctionsReport) {
val path = s"${config.sourceLocation.head}/${Constants.outputDirectoryName}"
UnresolvedReportUtility.reportUnresolvedMethods(xtocpg, path, Language.PYTHON)
}

// Run tagger
println(s"${Calendar.getInstance().getTime} - Tagging source code with rules...")
cpg.runTagger(processedRules)
Expand Down Expand Up @@ -153,10 +159,6 @@ object PythonProcessor {
)
cpg
}
if (config.showUnresolvedFunctionsReport) {
val path = s"${config.sourceLocation.head}/${Constants.outputDirectoryName}"
UnresolvedReportUtility.reportUnresolvedMethods(xtocpg, path, Language.PYTHON)
}
processCPG(xtocpg, processedRules, sourceRepoLocation)
}

Expand Down
32 changes: 16 additions & 16 deletions src/main/scala/ai/privado/model/Constants.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,22 +73,22 @@ object Constants {
val remoteUrl = "remoteUrl"
val endPoint = "endPoint"

val coreVersion = "privadoCoreVersion"
val cliVersion = "privadoCLIVersion"
val mainVersion = "privadoMainVersion"
val notDetected = "Not Detected"
val createdAt = "createdAt"
val repoName = "repoName"
val gitMetaData = "gitMetaData"
val sources = "sources"
val localScanPath = "localScanPath"
val processing = "processing"
val sinkProcessing = "sinkProcessing"
val probableSinks = "probableSinks"
val outputFileName = "privado.json"
val outputDirectoryName = ".privado"
val outputIntermediateFileName = "intermediate.json"

val coreVersion = "privadoCoreVersion"
val cliVersion = "privadoCLIVersion"
val mainVersion = "privadoMainVersion"
val notDetected = "Not Detected"
val createdAt = "createdAt"
val repoName = "repoName"
val gitMetaData = "gitMetaData"
val sources = "sources"
val localScanPath = "localScanPath"
val processing = "processing"
val sinkProcessing = "sinkProcessing"
val probableSinks = "probableSinks"
val outputFileName = "privado.json"
val outputDirectoryName = ".privado"
val outputIntermediateFileName = "intermediate.json"
val privadoLanguageEngineVersion = "privadoLanguageEngineVersion"
// database details
val dbName = "dbName"
val dbVendor = "dbVendor"
Expand Down
51 changes: 41 additions & 10 deletions src/main/scala/ai/privado/utility/UnresolvedReportUtility.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,18 @@ import ai.privado.model.Language
import scala.collection.mutable.ListBuffer
import io.shiftleft.semanticcpg.language._
import io.shiftleft.codepropertygraph.generated.Cpg
import ai.privado.utility.Utilities.getFileNameForNode
import ai.privado.utility.Utilities.{getFileNameForNode, resolver}

object UnresolvedReportUtility {
def reportUnresolvedMethods(xtocpg: Try[Cpg], statoutdir: String, language: Language.Language): Unit = {
var total = 0
var unresolvedSignatures = 0
var unresolvedNamespaces = 0
var unresolvedSignaturesList = ListBuffer[String]()
var unresolvedNamespacesList = ListBuffer[String]()
var total = 0
var unresolvedSignatures = 0
var unresolvedNamespaces = 0
var unresolvedSignatureWithCallee = 0
var unresolvedSignaturesList = ListBuffer[String]()
var unresolvedNamespacesList = ListBuffer[String]()
var nonempty = 0
var isempty = 0

val unresolved_signature = "(?i)(.*)(unresolved)(signature)(.*)"
val unresolved_namespace = "(?i)(.*)(unresolved)(namespace)(.*)"
Expand All @@ -52,15 +55,30 @@ object UnresolvedReportUtility {

xtocpg match {
case Success(cpg) => {
total = cpg.call.methodFullName.l.length
unresolvedSignatures = cpg.call.methodFullName(unresolved_sig_pattern).l.length
val importCount = cpg.call.l.filter((i) => i.name == "import").l.length
total = cpg.call.methodFullName.l.length - importCount
unresolvedSignatures = cpg.call.methodFullName(unresolved_sig_pattern).l.length - importCount

nonempty = cpg.call.callee.filter(_.nonEmpty == false).l.length
isempty = cpg.call.callee.filter(_.isEmpty).l.length

cpg.call
.methodFullName(unresolved_sig_pattern)
.l
.filter((i) => {
var res = true
if (language.equals(Language.PYTHON)) {
res = i.name != "import"
}
res
})
.map(us => {
unresolvedSignaturesList += us.methodFullName + "\n\t" + "Line Number: " + us.lineNumber.get + "\n\t" + "File: " + getFileNameForNode(
if (us.callee.fullName.l.length > 0) {
unresolvedSignatureWithCallee += 1
}
unresolvedSignaturesList += us.methodFullName + "(" + us.name + ")" + "\n\t" + "Line Number: " + us.lineNumber.get + "\n\t" + "File: " + getFileNameForNode(
us
)
) + "\n\t" + "Callee FullName: " + us.callee.fullName.l
})

if (language.equals(Language.JAVA)) {
Expand Down Expand Up @@ -101,6 +119,14 @@ object UnresolvedReportUtility {
statstr += s"$percentage% of total calls are unresolved\n"
}

if (language.equals(Language.PYTHON)) {
statstr += s"\nCalls with unresolved signatures having callee: $unresolvedSignatureWithCallee\n"
if (unresolvedSignatureWithCallee > 0) {
percentage = (unresolvedSignatureWithCallee.toDouble * 100.0) / unresolvedSignatures.toDouble
statstr += s"$percentage% of unresolved signatures having callee from unresolved signatures\n"
}
}

statstr += s"\nCalls with unresolved namespace: $unresolvedNamespaces\n"
if (unresolvedNamespaces > 0) {
percentage = (unresolvedNamespaces.toDouble * 100.0) / total.toDouble
Expand All @@ -118,6 +144,11 @@ object UnresolvedReportUtility {
print(statstr)
statfile.appendText(statstr)

// if (nonempty > 0)
statstr += s"\nCalls with nonEmpty Callee false: $nonempty"
// if (isempty > 0)
statstr += s"\nCalls with isEmpty Callee: $isempty\n"

if (unresolvedSignaturesList.length > 0) {
statfile.appendLine(divider)
statfile.appendLine("List of Calls with Unresolved Signatures:")
Expand Down

0 comments on commit 46605a9

Please sign in to comment.