From 0ab1978cbdb8acea810cef1bb457c475cef35af9 Mon Sep 17 00:00:00 2001 From: Pandurang Patil Date: Mon, 4 Dec 2023 15:51:54 +0530 Subject: [PATCH] [gosrc2cpg] - Handling of lambda TypeDecl (#3885) 1. Handling of lambda TypeDecl creation 2. Few additional tests. TODO: 1. Handling of global package level Contructor node. Along with verification of lambda defined at package level and the respective data flow. 2. Handling of lambda TypeDecl inheritance mapping with matching signture Struct types. 3. Handling of closure use case. 4. More data flow unit tests --- .../gosrc2cpg/astcreation/AstCreator.scala | 11 ++- .../astcreation/AstForLambdaCreator.scala | 22 +++--- .../passes/ast/AnonymousFuncTests.scala | 70 +++++++++++++++++-- .../TypeDeclMembersAndMemberMethodsTest.scala | 36 ++++++++++ 4 files changed, 123 insertions(+), 16 deletions(-) diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreator.scala index ebdfc178a912..1af8b3065987 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstCreator.scala @@ -6,8 +6,9 @@ import io.joern.gosrc2cpg.parser.ParserAst.* import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo} import io.joern.x2cpg.datastructures.Scope import io.joern.x2cpg.datastructures.Stack.* +import io.joern.x2cpg.utils.NodeBuilders.newModifierNode import io.joern.x2cpg.{Ast, AstCreatorBase, ValidationMode, AstNodeBuilder as X2CpgAstNodeBuilder} -import io.shiftleft.codepropertygraph.generated.NodeTypes +import io.shiftleft.codepropertygraph.generated.{ModifierTypes, NodeTypes} import io.shiftleft.codepropertygraph.generated.nodes.NewNode import org.slf4j.{Logger, LoggerFactory} import overflowdb.BatchedUpdate.DiffGraphBuilder @@ -74,7 +75,13 @@ class AstCreator(val relPathFileName: String, val parserResult: ParserResult, go .toList methodAstParentStack.pop() scope.popScope() - methodAst(fakeGlobalMethodForFile, Seq.empty, blockAst(blockNode_, declsAsts), methodReturn) + methodAst( + fakeGlobalMethodForFile, + Seq.empty, + blockAst(blockNode_, declsAsts), + methodReturn, + newModifierNode(ModifierTypes.MODULE) :: Nil + ) } protected def astForNode(nodeInfo: ParserNodeInfo, globalStatements: Boolean = false): Seq[Ast] = { diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForLambdaCreator.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForLambdaCreator.scala index a9ff37f0779c..0709a89156d2 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForLambdaCreator.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/astcreation/AstForLambdaCreator.scala @@ -2,10 +2,10 @@ package io.joern.gosrc2cpg.astcreation import io.joern.gosrc2cpg.datastructures.GoGlobal import io.joern.gosrc2cpg.parser.{ParserKeys, ParserNodeInfo} -import io.joern.x2cpg.{Ast, ValidationMode, Defines as XDefines} -import io.shiftleft.codepropertygraph.generated.nodes.{NewMethod, NewMethodReturn, NewTypeDecl} import io.joern.x2cpg.datastructures.Stack.StackWrapper import io.joern.x2cpg.utils.NodeBuilders.newModifierNode +import io.joern.x2cpg.{Ast, ValidationMode, Defines as XDefines} +import io.shiftleft.codepropertygraph.generated.nodes.{NewMethod, NewMethodReturn} import io.shiftleft.codepropertygraph.generated.{ModifierTypes, NodeTypes} import ujson.Value @@ -34,15 +34,17 @@ trait AstForLambdaCreator(implicit withSchemaValidation: ValidationMode) { this: ) scope.popScope() methodAstParentStack.pop() - baseFullName match - case fullyQualifiedPackage => - methodNode_.astParentType(NodeTypes.TYPE_DECL).astParentFullName(fullyQualifiedPackage) - case _ => - methodNode_.astParentType(NodeTypes.METHOD).astParentFullName(baseFullName) - Ast.storeInDiffGraph(astForMethod, diffGraph) - // TODO: Create TypeDecl for lambda function for which we didnt find the type. - // We need to create TypeDecl for every lambda function and set its inheritance with all the matching lambda types. + // TODO: We need to set the types defined for matching signature of the lambda as inehritied from // val typeFullName = GoGlobal.lambdaSignatureToLambdaTypeMap.getOrDefault(signature, fullName) + val typeDeclNode_ = typeDeclNode(funcLiteral, lambdaName, fullName, relPathFileName, lambdaName) + if baseFullName == fullyQualifiedPackage then + typeDeclNode_.astParentType(NodeTypes.TYPE_DECL).astParentFullName(fullyQualifiedPackage) + else typeDeclNode_.astParentType(NodeTypes.METHOD).astParentFullName(baseFullName) + Ast.storeInDiffGraph(Ast(typeDeclNode_), diffGraph) + // Setting Lambda TypeDecl as its parent. + methodNode_.astParentType(NodeTypes.TYPE_DECL) + methodNode_.astParentFullName(fullName) + Ast.storeInDiffGraph(astForMethod, diffGraph) GoGlobal.recordFullNameToReturnType(fullName, returnTypeStr, signature) Seq(Ast(methodRefNode(funcLiteral, funcLiteral.code, fullName, fullName))) } diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/AnonymousFuncTests.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/AnonymousFuncTests.scala index 9fa29ec44778..0e5c7e7dbdc5 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/AnonymousFuncTests.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/AnonymousFuncTests.scala @@ -1,9 +1,8 @@ package io.joern.go2cpg.passes.ast import io.joern.go2cpg.testfixtures.GoCodeToCpgSuite -import io.shiftleft.codepropertygraph.generated.{EvaluationStrategies, NodeTypes} -import io.shiftleft.semanticcpg.language.* import io.joern.x2cpg.Defines +import io.shiftleft.semanticcpg.language.* class AnonymousFuncTests extends GoCodeToCpgSuite { @@ -58,7 +57,70 @@ class AnonymousFuncTests extends GoCodeToCpgSuite { cpg.call("add").callee.fullName.l shouldBe List(s"main.main.${Defines.ClosurePrefix}0") } - // TODO: Add TypeDecl test - // TODO: Add unit test for lambda defined outside method and part of package directly + "have TypeDecl created for respective lambda" in { + cpg.typeDecl(s"${Defines.ClosurePrefix}0").fullName.l shouldBe List(s"main.main.${Defines.ClosurePrefix}0") + } + + "traversal from TypeDecl to lambda method" in { + cpg.typeDecl(s"${Defines.ClosurePrefix}0").method.fullName.l shouldBe List(s"main.main.${Defines.ClosurePrefix}0") + } + } + + "Simple Lambda expression defined in package" should { + // TODO: some of tests are ignore, which will require Struct Type constructor handling for the Package initialisation + val cpg = code(""" + |package main + | + |import "fmt" + | + |// Define a lambda function and assign it to a variable + |var add = func(a, b int) int { + | return a + b + |} + | + |func main() { + | + | // Call the lambda function + | result := add(3, 5) + | fmt.Println("Result:", result) // Output: 8 + |} + |""".stripMargin) + "have proper methodRef node created along with its properties" ignore { + cpg.methodRef.l.size shouldBe 1 + val List(mr) = cpg.methodRef.l + mr.methodFullName shouldBe s"main.${Defines.ClosurePrefix}0" + mr.typeFullName shouldBe s"main.${Defines.ClosurePrefix}0" + } + + "have proper Method node created along with its properties" ignore { + cpg.method.isLambda.l.size shouldBe 1 + val List(m) = cpg.method.isLambda.l + m.fullName shouldBe s"main.${Defines.ClosurePrefix}0" + m.signature shouldBe s"${Defines.ClosurePrefix}(int, int)int" + } + + "able to traverse to referenced Method node" ignore { + cpg.methodRef.referencedMethod.fullName.l shouldBe List(s"main.${Defines.ClosurePrefix}0") + } + + "reflects into lhs side TypeFullName" ignore { + cpg.local("add").typeFullName.l shouldBe List(s"main.${Defines.ClosurePrefix}0") + } + + "have call node created for lambda invocation" ignore { + cpg.call("add").methodFullName.l shouldBe List(s"main.${Defines.ClosurePrefix}0") + } + + "able to traverse from call node to callee" ignore { + cpg.call("add").callee.fullName.l shouldBe List(s"main.${Defines.ClosurePrefix}0") + } + + "have TypeDecl created for respective lambda" in { + cpg.typeDecl(s"${Defines.ClosurePrefix}0").fullName.l shouldBe List(s"main.${Defines.ClosurePrefix}0") + } + + "traversal from TypeDecl to lambda method" in { + cpg.typeDecl(s"${Defines.ClosurePrefix}0").method.fullName.l shouldBe List(s"main.${Defines.ClosurePrefix}0") + } } } diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/TypeDeclMembersAndMemberMethodsTest.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/TypeDeclMembersAndMemberMethodsTest.scala index 08af85d830bf..878af00ed211 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/TypeDeclMembersAndMemberMethodsTest.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/TypeDeclMembersAndMemberMethodsTest.scala @@ -236,4 +236,40 @@ class TypeDeclMembersAndMemberMethodsTest extends GoCodeToCpgSuite { areaByReferenceNode.fullName shouldBe "main.StringAlias.SomeMethodOne" } } + + "Member method of Struct type defined in another file but in the same package" should { + val cpg = code( + """ + |module joern.io/sample + |go 1.18 + |""".stripMargin, + "go.mod" + ).moreCode( + """ + |package main + |type Sample struct { + | name int + |} + |""".stripMargin, + "lib.go" + ).moreCode( + """ + |package main + |func (sample Sample) someProcessing() int{ + | return 0 + |} + |func foo(argc int, argv Sample) { + |} + |""".stripMargin, + "main.go" + ) + + "Have typeDecl created " in { + cpg.typeDecl("Sample").fullName.l shouldBe List("main.Sample") + } + + "Have member method part of typeDecl" in { + cpg.typeDecl("Sample").method.fullName.l shouldBe List("main.Sample.someProcessing") + } + } }