Skip to content

Commit

Permalink
Merge pull request #488 from Privado-Inc/dev
Browse files Browse the repository at this point in the history 
add - soap api and collection point where method is denoted by webser…
  • Loading branch information
@khemrajrathore
khemrajrathore authored Apr 6, 2023
2 parents d115ff7 + 37544d3 commit 37ac848
Show file tree
Hide file tree
Showing 17 changed files with 630 additions and 108 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
Privado Core
=============================================

Branch structure
Branch structure

main - This branch will contain the released version of the code.

Expand Down
42 changes: 42 additions & 0 deletions src/main/scala/ai/privado/audit/AuditReportConstants.scala
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package ai.privado.audit

object AuditReportConstants {

val AUDIT_ELEMENT_DISCOVERY_SHEET_NAME = "Element-Discovery"

val AUDIT_EMPTY_CELL_VALUE = "--"

val AUDIT_CHECKED_VALUE = "YES"

val AUDIT_NOT_CHECKED_VALUE = "NO"

val AUDIT_BUILT_IN_CLASS_REGEX =
"\\b((int|byte|short|long|float|double|boolean|char)|((java\\.|javax\\.|org\\.|com\\.sun\\.|com\\.oracle\\.)([A-Z][a-zA-Z0-9_]*(\\.[A-Z][a-zA-Z0-9_]*)*)))\\b"

val ELEMENT_DISCOVERY_CLASS_NAME = "Class"

val ELEMENT_DISCOVERY_FILE_NAME = "File Name"

val ELEMENT_DISCOVERY_MEMBER_NAME = "Member"

val ELEMENT_DISCOVERY_MEMBER_TYPE = "Member Type"

val ELEMENT_DISCOVERY_TAGGED_NAME = "Tagged"

val ELEMENT_DISCOVERY_SOURCE_RULE_ID = "Source Rule ID"

val ELEMENT_DISCOVERY_INPUT_COLLECTION = "Input to Collection"

val ELEMENT_DISCOVERY_COLLECTION_ENDPOINT = "Collection Endpoint Path"

val ELEMENT_DISCOVERY_METHOD_NAME = "Collection Method Full Name"

val ELEMENT_DISCOVERY_EXCLUDE_CLASS_NAME_REGEX = "^(.*)(Controller|Service|Impl|Helper|Util|Processor|Dao)$"

val ELEMENT_DISCOVERY_GET_SET_METHOD_REGEX = "^(get|set).*"

val ELEMENT_DISCOVERY_OVERRIDE_METHOD_REGEX = "^(hascode|equals)"

val ELEMENT_DISCOVERY_GETTER_SETTER_REGEX = ".*(Getter|Setter).*"

}
183 changes: 154 additions & 29 deletions src/main/scala/ai/privado/audit/DataElementDiscovery.scala
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
package ai.privado.audit

import ai.privado.cache.TaggerCache
import ai.privado.model.{CatLevelOne, Constants, InternalTag}
import io.shiftleft.codepropertygraph.generated.Cpg
import io.shiftleft.codepropertygraph.generated.nodes.{Member, MethodParameterIn}
import io.shiftleft.codepropertygraph.generated.nodes.{Member, TypeDecl}
import io.shiftleft.semanticcpg.language._
import org.slf4j.LoggerFactory

Expand All @@ -14,8 +15,6 @@ object DataElementDiscovery {

private val logger = LoggerFactory.getLogger(getClass)

private val excludeClassNameRegex = "^(.*)(Controller|Service|Impl|Helper|Util|Processor)$"

// Get list of Class Name having getter and setter method
def getSourceUsingRules(xtocpg: Try[Cpg]): List[String] = {
logger.info("Process Class Name from cpg")
Expand All @@ -25,11 +24,11 @@ object DataElementDiscovery {
// Get DTO/Entity Class name
val typeDeclList = cpg.typeDecl
.filter(_.order > 0)
.whereNot(_.name(excludeClassNameRegex))
.whereNot(_.name(AuditReportConstants.ELEMENT_DISCOVERY_EXCLUDE_CLASS_NAME_REGEX))
.or(
_.where(_.method.name("^(get|set).*")),
_.where(_.method.name("^(hascode|equals)")),
_.where(_.annotation.name(".*(Getter|Setter).*"))
_.where(_.method.name(AuditReportConstants.ELEMENT_DISCOVERY_GET_SET_METHOD_REGEX)),
_.where(_.method.name(AuditReportConstants.ELEMENT_DISCOVERY_OVERRIDE_METHOD_REGEX)),
_.where(_.annotation.name(AuditReportConstants.ELEMENT_DISCOVERY_GETTER_SETTER_REGEX))
)
.toList
typeDeclList.foreach(node => {
Expand Down Expand Up @@ -68,7 +67,7 @@ object DataElementDiscovery {
val typeDeclList = cpg.typeDecl
.filter(_.order > 0)
.where(_.fullName(pattern))
.whereNot(_.name(excludeClassNameRegex))
.whereNot(_.name(AuditReportConstants.ELEMENT_DISCOVERY_EXCLUDE_CLASS_NAME_REGEX))
.toList
typeDeclList.foreach(typeDecl => derivedClassName += typeDecl.fullName)
})
Expand All @@ -83,16 +82,16 @@ object DataElementDiscovery {
}

// Get list of member variable present in given class
def getMemberUsingClassName(xtocpg: Try[Cpg], classNameSet: Set[String]): mutable.Map[String, List[Member]] = {
def getMemberUsingClassName(xtocpg: Try[Cpg], classNameSet: Set[String]): Map[TypeDecl, List[Member]] = {
logger.info("Process Member Name from cpg")
val memberInfoMap = mutable.Map[String, List[Member]]()
val memberInfoMap = mutable.HashMap[TypeDecl, List[Member]]()

xtocpg match {
case Success(cpg) => {
classNameSet.foreach(className => {
// Get member variable of class and put it into map
val memberInfo = cpg.typeDecl.filter(_.order > 0).where(_.fullName(className)).member.toList
memberInfoMap.put(className, memberInfo)
cpg.typeDecl
.where(_.fullName(className))
.foreach(typeDeclNode => memberInfoMap.put(typeDeclNode, typeDeclNode.member.l))
})
}
case Failure(exception) => {
Expand All @@ -102,7 +101,7 @@ object DataElementDiscovery {
}
}
logger.info("Successfully Processed member name from cpg")
memberInfoMap
memberInfoMap.toMap
}

// Get Collection Input Class Name
Expand All @@ -112,7 +111,11 @@ object DataElementDiscovery {
xtocpg match {
case Success(cpg) => {
// Get tagged collection input list
val parameterList = cpg.parameter.where(_.tag).l
val parameterList = cpg.parameter
.where(_.method.tag.nameExact(Constants.catLevelOne).valueExact(CatLevelOne.COLLECTIONS.name))
.whereNot(_.typeFullName(AuditReportConstants.AUDIT_BUILT_IN_CLASS_REGEX))
.whereNot(_.typeFullName(AuditReportConstants.ELEMENT_DISCOVERY_EXCLUDE_CLASS_NAME_REGEX))
.l
parameterList.foreach(parameter => {
collectionInputList += parameter.typeFullName
})
Expand All @@ -126,11 +129,45 @@ object DataElementDiscovery {
collectionInputList.toList
}

def getCollectionMethodInfo(xtocpg: Try[Cpg]): Map[String, ListBuffer[CollectionMethodInfo]] = {
// className -> (MethodName, Endpoint)
val collectionMethodInfoMap = new mutable.HashMap[String, ListBuffer[CollectionMethodInfo]]()
xtocpg match {
case Success(cpg) => {
val parameterList = cpg.parameter
.where(_.method.tag.nameExact(Constants.catLevelOne).valueExact(CatLevelOne.COLLECTIONS.name))
.whereNot(_.typeFullName(AuditReportConstants.AUDIT_BUILT_IN_CLASS_REGEX))
.whereNot(_.typeFullName(AuditReportConstants.ELEMENT_DISCOVERY_EXCLUDE_CLASS_NAME_REGEX))
.l
parameterList.foreach(parameter => {

// Get complete Endpoint path
val endpointTag = parameter.method.tag.where(_.name(InternalTag.COLLECTION_METHOD_ENDPOINT.toString)).head
// Append endpoint and method name when Method parameter having same type
if (!collectionMethodInfoMap.contains(parameter.typeFullName)) {
collectionMethodInfoMap
.put(parameter.typeFullName, new ListBuffer[CollectionMethodInfo])
}

val collectionMethodInfo = collectionMethodInfoMap(parameter.typeFullName)
collectionMethodInfo += CollectionMethodInfo(parameter.method.code, endpointTag.value)
})
}
case Failure(exception) => {
println("Failed to process collection method info from cpg")
logger.debug("Failed to process collection method info from cpg", exception)
println(exception.printStackTrace())
}
}
collectionMethodInfoMap.toMap
}

def processDataElementDiscovery(xtocpg: Try[Cpg], taggerCache: TaggerCache): List[List[String]] = {
logger.info("Initiated the audit engine")
val classNameRuleList = getSourceUsingRules(xtocpg)
val collectionInputList = getCollectionInputList(xtocpg)
val derivedClassName = extractClassFromPackage(xtocpg, (classNameRuleList ++ collectionInputList).toSet)
val classNameRuleList = getSourceUsingRules(xtocpg)
val collectionInputList = getCollectionInputList(xtocpg)
val collectionMethodInfo = getCollectionMethodInfo(xtocpg)
val derivedClassName = extractClassFromPackage(xtocpg, (classNameRuleList ++ collectionInputList).toSet)
val memberInfo =
getMemberUsingClassName(xtocpg, (classNameRuleList ++ collectionInputList ++ derivedClassName).toSet)
val workbookResult = new ListBuffer[List[String]]()
Expand All @@ -151,33 +188,119 @@ object DataElementDiscovery {
}

// Header List
workbookResult += List("Class", "Member", "Tagged", "Source Rule ID", "Input to Collection")
workbookResult += List(
AuditReportConstants.ELEMENT_DISCOVERY_CLASS_NAME,
AuditReportConstants.ELEMENT_DISCOVERY_FILE_NAME,
AuditReportConstants.ELEMENT_DISCOVERY_MEMBER_NAME,
AuditReportConstants.ELEMENT_DISCOVERY_MEMBER_TYPE,
AuditReportConstants.ELEMENT_DISCOVERY_TAGGED_NAME,
AuditReportConstants.ELEMENT_DISCOVERY_SOURCE_RULE_ID,
AuditReportConstants.ELEMENT_DISCOVERY_INPUT_COLLECTION,
AuditReportConstants.ELEMENT_DISCOVERY_COLLECTION_ENDPOINT,
AuditReportConstants.ELEMENT_DISCOVERY_METHOD_NAME
)

// Construct the excel sheet and fill the data
try {
memberInfo.foreach {
case (key, value) => {
val isCollectionInput = if (collectionInputList.contains(key)) "YES" else "NO"
if (taggedMemberInfo.contains(key)) {
workbookResult += List(key, "--", "YES", "", isCollectionInput)
val ruleMemberInfo = taggedMemberInfo.getOrElse(key, new mutable.HashMap[String, String])
val isCollectionInput = if (collectionInputList.contains(key.fullName)) "YES" else "NO"
if (taggedMemberInfo.contains(key.fullName)) {
if (collectionMethodInfo.contains(key.fullName)) {
collectionMethodInfo(key.fullName).foreach(info => {
workbookResult += List(
key.fullName,
key.file.head.name,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
AuditReportConstants.AUDIT_CHECKED_VALUE,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
isCollectionInput,
info.endpoint,
info.methodDetail
)
})
} else {
workbookResult += List(
key.fullName,
key.file.head.name,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
AuditReportConstants.AUDIT_CHECKED_VALUE,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
isCollectionInput,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE
)
}
val ruleMemberInfo = taggedMemberInfo.getOrElse(key.fullName, new mutable.HashMap[String, String])
value.foreach(member => {
if (ruleMemberInfo.contains(member.name)) {
workbookResult += List(
"",
key.fullName,
key.file.head.name,
member.name,
"YES",
member.typeFullName,
AuditReportConstants.AUDIT_CHECKED_VALUE,
ruleMemberInfo.getOrElse(member.name, "Default value"),
""
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE
)
} else {
workbookResult += List("", member.name, "NO", "--", "")
workbookResult += List(
key.fullName,
key.file.head.name,
member.name,
member.typeFullName,
AuditReportConstants.AUDIT_NOT_CHECKED_VALUE,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE
)
}
})
} else {
workbookResult += List(key, "--", "NO", "", isCollectionInput)
if (collectionMethodInfo.contains(key.fullName)) {
collectionMethodInfo(key.fullName).foreach(info => {
workbookResult += List(
key.fullName,
key.file.head.name,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
AuditReportConstants.AUDIT_NOT_CHECKED_VALUE,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
isCollectionInput,
info.endpoint,
info.methodDetail
)
})
} else {
workbookResult += List(
key.fullName,
key.file.head.name,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
AuditReportConstants.AUDIT_NOT_CHECKED_VALUE,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
isCollectionInput,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE
)
}
value.foreach(member => {
workbookResult += List("", member.name, "NO", "--", "")
workbookResult += List(
key.fullName,
key.file.head.name,
member.name,
member.typeFullName,
AuditReportConstants.AUDIT_NOT_CHECKED_VALUE,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
AuditReportConstants.AUDIT_EMPTY_CELL_VALUE
)
})
}
}
Expand All @@ -190,4 +313,6 @@ object DataElementDiscovery {
}
workbookResult.toList
}

case class CollectionMethodInfo(var methodDetail: String, var endpoint: String)
}
31 changes: 28 additions & 3 deletions src/main/scala/ai/privado/exporter/ExcelExporter.scala
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
package ai.privado.exporter

import ai.privado.audit.AuditReportConstants
import ai.privado.model.Constants.outputDirectoryName
import better.files.File
import org.apache.poi.ss.usermodel.{Cell, Row, Sheet, Workbook}
import org.apache.poi.xssf.usermodel.XSSFWorkbook
import org.apache.poi.ss.usermodel.{Cell, FillPatternType, IndexedColors, Row, Sheet, Workbook}
import org.apache.poi.xssf.usermodel.{XSSFCellStyle, XSSFColor, XSSFWorkbook}
import org.slf4j.LoggerFactory

import java.io.FileOutputStream
Expand All @@ -17,7 +18,7 @@ object ExcelExporter {
logger.info("Initiated the Audit exporter engine")
try {
val workbook: Workbook = new XSSFWorkbook()
val sheet: Sheet = workbook.createSheet("Member-Tag")
val sheet: Sheet = workbook.createSheet(AuditReportConstants.AUDIT_ELEMENT_DISCOVERY_SHEET_NAME)

// Iterate over the data and write it to the sheet
for ((rowValues, rowIndex) <- output.zipWithIndex) {
Expand All @@ -27,6 +28,9 @@ object ExcelExporter {
cell.setCellValue(cellValue)
}
}

// Changed Background colour when tagged
changeTaggedBackgroundColour(workbook, List(4, 6))
logger.info("Successfully added audit report to excel file")

// create directory if not exist
Expand All @@ -46,4 +50,25 @@ object ExcelExporter {
Left(ex.toString)
}
}

private def changeTaggedBackgroundColour(workbook: Workbook, columnList: List[Integer]) = {

val sheet = workbook.getSheet(AuditReportConstants.AUDIT_ELEMENT_DISCOVERY_SHEET_NAME)

val greenCellStyle: XSSFCellStyle = workbook.createCellStyle().asInstanceOf[XSSFCellStyle]
val greenColor: XSSFColor = new XSSFColor(IndexedColors.LIGHT_GREEN, null)
greenCellStyle.setFillForegroundColor(greenColor)
greenCellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND)

val rowIterator = sheet.rowIterator()
while (rowIterator.hasNext) {
val row = rowIterator.next()
columnList.foreach(columnNo => {
val cell = row.getCell(columnNo)
if (cell != null && cell.getStringCellValue == AuditReportConstants.AUDIT_CHECKED_VALUE) {
cell.setCellStyle(greenCellStyle)
}
})
}
}
}
2 changes: 1 addition & 1 deletion src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ object JavaProcessor {
// Run tagger
println(s"${Calendar.getInstance().getTime} - Tagging source code with rules...")
val taggerCache = new TaggerCache
cpg.runTagger(processedRules, taggerCache)
cpg.runTagger(processedRules, taggerCache, ScanProcessor.config)
println(
s"${TimeMetric.getNewTime()} - Tagging source code is done in \t\t\t- ${TimeMetric.setNewTimeToLastAndGetTimeDiff()}"
)
Expand Down
Loading

0 comments on commit 37ac848

Please sign in to comment.