diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DeclarationHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DeclarationHandler.kt index 8b13c5261b..83bfe96c4d 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DeclarationHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DeclarationHandler.kt @@ -34,7 +34,6 @@ import de.fraunhofer.aisec.cpg.graph.Annotation import de.fraunhofer.aisec.cpg.graph.declarations.* import de.fraunhofer.aisec.cpg.graph.scopes.FunctionScope import de.fraunhofer.aisec.cpg.graph.scopes.RecordScope -import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression import de.fraunhofer.aisec.cpg.graph.types.FunctionType @@ -48,34 +47,15 @@ import de.fraunhofer.aisec.cpg.helpers.Util * [DeclarationHandler], for others, the [StatementHandler] will forward these statements to us. */ class DeclarationHandler(frontend: PythonLanguageFrontend) : - PythonHandler(::ProblemDeclaration, frontend) { - override fun handleNode(node: Python.AST.BaseStmt): Declaration { + PythonHandler(::ProblemDeclaration, frontend) { + override fun handleNode(node: Python.AST.Def): Declaration { return when (node) { is Python.AST.FunctionDef -> handleFunctionDef(node) is Python.AST.AsyncFunctionDef -> handleFunctionDef(node) is Python.AST.ClassDef -> handleClassDef(node) - else -> { - return handleNotSupported(node, node.javaClass.simpleName) - } } } - private fun handleNotSupported(node: Python.AST.BaseStmt, name: String): Declaration { - Util.errorWithFileLocation( - frontend, - node, - log, - "Parsing of type $name is not supported (yet)", - ) - - val cpgNode = this.configConstructor.get() - if (cpgNode is ProblemNode) { - cpgNode.problem = "Parsing of type $name is not supported (yet)" - } - - return cpgNode - } - /** * Translates a Python [`ClassDef`](https://docs.python.org/3/library/ast.html#ast.ClassDef) * into an [RecordDeclaration]. @@ -95,7 +75,7 @@ class DeclarationHandler(frontend: PythonLanguageFrontend) : when (s) { // In order to be as compatible as possible with existing languages, we try to add // declarations directly to the class - is Python.AST.WithDeclaration -> handle(s) + is Python.AST.Def -> handle(s) // All other statements are added to the statements block of the class else -> cls.statements += frontend.statementHandler.handle(s) } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt index be9d163926..bb5490ea03 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg.frontends.python import de.fraunhofer.aisec.cpg.graph.declarations.Declaration +import de.fraunhofer.aisec.cpg.graph.statements.Statement import jep.python.PyObject /** @@ -59,13 +60,20 @@ interface Python { */ interface AST { + /** + * Represents a `ast.AST` node as returned by Python's `ast` parser. + * + * @param pyObject The Python object returned by jep. + */ + interface AST { + var pyObject: PyObject + } + /** * Some nodes, such as `ast.stmt` [AST.BaseStmt] and `ast.expr` [AST.BaseExpr] nodes have * extra location properties as implemented here. */ - interface WithLocation { // TODO make the fields accessible `by lazy` - val pyObject: PyObject - + interface WithLocation : AST { // TODO make the fields accessible `by lazy` /** Maps to the `lineno` filed from Python's ast. */ val lineno: Int get() { @@ -92,21 +100,15 @@ interface Python { } /** - * Python does not really have "declarations", but rather these are also [AST.BaseStmt]s. - * But in order to be compatible with the remaining languages we need to ensure that - * elements such as functions or classes, still turn out to be [Declaration]s. + * Python does not really have "declarations", but it has "definitions". Instead of having + * their own AST class, they are also [AST.BaseStmt]s. In order to be compatible with the + * remaining languages we need to ensure that elements such as functions or classes, still + * turn out to be [Declaration]s, not [Statement]s * * This interface should be attached to all such statements that we consider to be - * declarations. - */ - sealed interface WithDeclaration - - /** - * Represents a `ast.AST` node as returned by Python's `ast` parser. - * - * @param pyObject The Python object returned by jep. + * definitions, and thus [Declaration]s. */ - abstract class AST(pyObject: PyObject) : BaseObject(pyObject) + sealed interface Def : AST /** * ``` @@ -119,7 +121,7 @@ interface Python { * * Note: We currently only support `Module`s. */ - abstract class BaseMod(pyObject: PyObject) : AST(pyObject) + abstract class BaseMod(pyObject: PyObject) : AST, BaseObject(pyObject) /** * ``` @@ -127,7 +129,7 @@ interface Python { * | Module(stmt* body, type_ignore* type_ignores) * ``` */ - class Module(pyObject: PyObject) : AST(pyObject) { + class Module(pyObject: PyObject) : AST, BaseObject(pyObject) { val body: kotlin.collections.List by lazy { "body" of pyObject } val type_ignores: kotlin.collections.List by lazy { @@ -167,7 +169,7 @@ interface Python { * | | Continue * ``` */ - sealed class BaseStmt(pyObject: PyObject) : AST(pyObject), WithLocation + sealed class BaseStmt(pyObject: PyObject) : AST, BaseObject(pyObject), WithLocation /** * Several classes are duplicated in the python AST for async and non-async variants. This @@ -183,7 +185,7 @@ interface Python { * However, they are so similar, that we make use of this interface to avoid a lot of * duplicate code. */ - sealed interface NormalOrAsyncFunctionDef : AsyncOrNot, WithDeclaration { + sealed interface NormalOrAsyncFunctionDef : AsyncOrNot, Def { val name: String val args: arguments val body: kotlin.collections.List @@ -243,7 +245,7 @@ interface Python { * | ClassDef(identifier name, expr* bases, keyword* keywords, stmt* body, expr* decorator_list) * ``` */ - class ClassDef(pyObject: PyObject) : BaseStmt(pyObject), WithDeclaration { + class ClassDef(pyObject: PyObject) : BaseStmt(pyObject), Def { val name: String by lazy { "name" of pyObject } val bases: kotlin.collections.List by lazy { "bases" of pyObject } @@ -564,7 +566,7 @@ interface Python { * * ast.expr = class expr(AST) */ - sealed class BaseExpr(pyObject: PyObject) : AST(pyObject), WithLocation + sealed class BaseExpr(pyObject: PyObject) : AST, BaseObject(pyObject), WithLocation /** * ``` @@ -883,7 +885,7 @@ interface Python { * | boolop = And | Or * ``` */ - sealed class BaseBoolOp(pyObject: PyObject) : AST(pyObject) + sealed class BaseBoolOp(pyObject: PyObject) : AST, BaseObject(pyObject) /** * ``` @@ -906,7 +908,7 @@ interface Python { * | cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn * ``` */ - sealed class BaseCmpOp(pyObject: PyObject) : AST(pyObject) + sealed class BaseCmpOp(pyObject: PyObject) : AST, BaseObject(pyObject) /** * ``` @@ -994,7 +996,7 @@ interface Python { * | expr_context = Load | Store | Del * ``` */ - sealed class BaseExprContext(pyObject: PyObject) : AST(pyObject) + sealed class BaseExprContext(pyObject: PyObject) : AST, BaseObject(pyObject) /** * ``` @@ -1026,7 +1028,7 @@ interface Python { * | operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift | RShift | BitOr | BitXor | BitAnd | FloorDiv * ``` */ - sealed class BaseOperator(pyObject: PyObject) : AST(pyObject) + sealed class BaseOperator(pyObject: PyObject) : AST, BaseObject(pyObject) /** * ``` @@ -1145,7 +1147,7 @@ interface Python { * | | MatchOr(pattern* patterns) * ``` */ - abstract class BasePattern(pyObject: PyObject) : AST(pyObject), WithLocation + abstract class BasePattern(pyObject: PyObject) : AST, BaseObject(pyObject), WithLocation /** * ``` @@ -1246,7 +1248,7 @@ interface Python { * | unaryop = Invert | Not | UAdd | USub * ``` */ - sealed class BaseUnaryOp(pyObject: PyObject) : AST(pyObject) + sealed class BaseUnaryOp(pyObject: PyObject) : AST, BaseObject(pyObject) /** * ``` @@ -1286,7 +1288,7 @@ interface Python { * | alias(identifier name, identifier? asname) * ``` */ - class alias(pyObject: PyObject) : AST(pyObject), WithLocation { + class alias(pyObject: PyObject) : AST, BaseObject(pyObject), WithLocation { val name: String by lazy { "name" of pyObject } val asname: String? by lazy { "asname" of pyObject } } @@ -1297,7 +1299,7 @@ interface Python { * | arg(identifier arg, expr? annotation, string? type_comment) * ``` */ - class arg(pyObject: PyObject) : AST(pyObject), WithLocation { + class arg(pyObject: PyObject) : AST, BaseObject(pyObject), WithLocation { val arg: String by lazy { "arg" of pyObject } val annotation: BaseExpr? by lazy { "annotation" of pyObject } val type_comment: String? by lazy { "type_comment" of pyObject } @@ -1309,7 +1311,7 @@ interface Python { * | arguments(arg* posonlyargs, arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, arg? kwarg, expr* defaults) * ``` */ - class arguments(pyObject: PyObject) : AST(pyObject) { + class arguments(pyObject: PyObject) : AST, BaseObject(pyObject) { val posonlyargs: kotlin.collections.List by lazy { "posonlyargs" of pyObject } val args: kotlin.collections.List by lazy { "args" of pyObject } val vararg: arg? by lazy { "vararg" of pyObject } @@ -1325,7 +1327,7 @@ interface Python { * | comprehension(expr target, expr iter, expr* ifs, int is_async) * ``` */ - class comprehension(pyObject: PyObject) : AST(pyObject) { + class comprehension(pyObject: PyObject) : AST, BaseObject(pyObject) { val target: BaseExpr by lazy { "target" of pyObject } val iter: BaseExpr by lazy { "iter" of pyObject } val ifs: kotlin.collections.List by lazy { "ifs" of pyObject } @@ -1338,7 +1340,8 @@ interface Python { * | excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) * ``` */ - sealed class BaseExcepthandler(python: PyObject) : AST(python), WithLocation + sealed class BaseExcepthandler(pyObject: PyObject) : + AST, BaseObject(pyObject), WithLocation /** * ast.ExceptHandler = class ExceptHandler(excepthandler) | ExceptHandler(expr? type, @@ -1356,7 +1359,7 @@ interface Python { * | keyword(identifier? arg, expr value) * ``` */ - class keyword(pyObject: PyObject) : AST(pyObject), WithLocation { + class keyword(pyObject: PyObject) : AST, BaseObject(pyObject), WithLocation { val arg: String? by lazy { "arg" of pyObject } val value: BaseExpr by lazy { "value" of pyObject } } @@ -1367,7 +1370,7 @@ interface Python { * | match_case(pattern pattern, expr? guard, stmt* body) * ``` */ - class match_case(pyObject: PyObject) : AST(pyObject) { + class match_case(pyObject: PyObject) : AST, BaseObject(pyObject) { val pattern: BasePattern by lazy { "pattern" of pyObject } val guard: BaseExpr? by lazy { "guard" of pyObject } val body: kotlin.collections.List by lazy { "body" of pyObject } @@ -1381,7 +1384,7 @@ interface Python { * * TODO */ - class type_ignore(pyObject: PyObject) : AST(pyObject) + class type_ignore(pyObject: PyObject) : AST, BaseObject(pyObject) /** * ``` @@ -1389,7 +1392,7 @@ interface Python { * | withitem(expr context_expr, expr? optional_vars) * ``` */ - class withitem(pyObject: PyObject) : AST(pyObject) { + class withitem(pyObject: PyObject) : AST, BaseObject(pyObject) { val context_expr: BaseExpr by lazy { "context_expr" of pyObject } val optional_vars: BaseExpr? by lazy { "optional_vars" of pyObject } } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt index ecdeec3ccc..2179188b22 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt @@ -349,7 +349,7 @@ class PythonLanguageFrontend(language: Language, ctx: Tr when (stmt) { // In order to be as compatible as possible with existing languages, we try to // add declarations directly to the class - is Python.AST.WithDeclaration -> declarationHandler.handle(stmt) + is Python.AST.Def -> declarationHandler.handle(stmt) // All other statements are added to the (static) statements block of the // namespace. else -> lastNamespace.statements += statementHandler.handle(stmt) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index ceb2c51558..54f51d95a7 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -75,7 +75,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : problem = "The statement of class ${node.javaClass} is not supported yet", rawNode = node, ) - is Python.AST.WithDeclaration -> + is Python.AST.Def -> wrapDeclarationToStatement(frontend.declarationHandler.handleNode(node)) } } @@ -270,10 +270,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : } /** Prepares the `manager.__exit__(None, None, None)` call for the else-block. */ - fun generateExitCallWithNone( - managerName: Name, - withItem: Python.AST.withitem, - ): MemberCallExpression { + fun generateExitCallWithNone(managerName: Name): MemberCallExpression { val exitCallWithNone = newMemberCallExpression( callee = @@ -295,10 +292,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : * Prepares the if-statement which is the body of the catch block. This includes the call of * `manager.__exit__(*sys.exc_info())`, the negation and the throw statement. */ - fun generateExitCallWithSysExcInfo( - managerName: Name, - withItem: Python.AST.withitem, - ): IfStatement { + fun generateExitCallWithSysExcInfo(managerName: Name): IfStatement { val exitCallWithSysExec = newMemberCallExpression( callee = @@ -331,10 +325,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : * tmpVal = manager.__enter__() * ``` */ - fun generateEnterCallAndAssignment( - managerName: Name, - withItem: Python.AST.withitem, - ): Pair { + fun generateEnterCallAndAssignment(managerName: Name): Pair { val tmpValName = Name.random(prefix = WITH_TMP_VAL) val enterVar = newReference(name = tmpValName).implicit() val enterCall = @@ -373,8 +364,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : currentBlock.statements.add(managerAssignment) - val (enterAssignment, tmpValName) = - generateEnterCallAndAssignment(managerName, withItem) + val (enterAssignment, tmpValName) = generateEnterCallAndAssignment(managerName) currentBlock.statements.add(enterAssignment) // Create the try statement with __exit__ calls in the finally block @@ -416,7 +406,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : this.body = newBlock().implicit().apply { this.statements.add( - generateExitCallWithSysExcInfo(managerName, withItem) + generateExitCallWithSysExcInfo(managerName) ) } } @@ -424,7 +414,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : // Add the else-block this.elseBlock = newBlock().implicit().apply { - this.statements.add(generateExitCallWithNone(managerName, withItem)) + this.statements.add(generateExitCallWithNone(managerName)) } } currentBlock.statements.add(tryStatement) @@ -846,25 +836,21 @@ class StatementHandler(frontend: PythonLanguageFrontend) : * [parentNode]. */ internal fun makeBlock( - stmts: List, + statements: List, parentNode: Python.AST.WithLocation, ): Block { val result = newBlock() - for (stmt in stmts) { + for (stmt in statements) { result.statements += handle(stmt) } - // Try to retrieve the code and location from the parent node, if it is a base stmt - val ast = parentNode as? Python.AST.AST - if (ast != null) { - // We need to scope the call to codeAndLocationFromChildren to our frontend, so that - // all Python.AST.AST nodes are accepted, otherwise it would be scoped to the handler - // and only Python.AST.BaseStmt nodes would be accepted. This would cause issues with - // other nodes that are not "statements", but also handled as part of this handler, - // e.g., the Python.AST.ExceptHandler. - with(frontend) { result.codeAndLocationFromChildren(ast, frontend.lineSeparator) } - } + // We need to scope the call to codeAndLocationFromChildren to our frontend, so that + // all Python.AST.AST nodes are accepted, otherwise it would be scoped to the handler + // and only Python.AST.BaseStmt nodes would be accepted. This would cause issues with + // other nodes that are not "statements", but also handled as part of this handler, + // e.g., the Python.AST.ExceptHandler. + with(frontend) { result.codeAndLocationFromChildren(parentNode, frontend.lineSeparator) } return result }