Some more cleanup and trying to harmonize AST interfaces

Fraunhofer-AISEC · Feb 11, 2025 · 514c8f7 · 514c8f7
1 parent 415479f
commit 514c8f7
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 89 deletions.
diff --git a/...age-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DeclarationHandler.kt b/...age-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DeclarationHandler.kt
@@ -34,7 +34,6 @@ import de.fraunhofer.aisec.cpg.graph.Annotation
 import de.fraunhofer.aisec.cpg.graph.declarations.*
 import de.fraunhofer.aisec.cpg.graph.scopes.FunctionScope
 import de.fraunhofer.aisec.cpg.graph.scopes.RecordScope
-import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression
 import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression
 import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression
 import de.fraunhofer.aisec.cpg.graph.types.FunctionType
@@ -48,34 +47,15 @@ import de.fraunhofer.aisec.cpg.helpers.Util
  * [DeclarationHandler], for others, the [StatementHandler] will forward these statements to us.
  */
 class DeclarationHandler(frontend: PythonLanguageFrontend) :
-    PythonHandler<Declaration, Python.AST.BaseStmt>(::ProblemDeclaration, frontend) {
-    override fun handleNode(node: Python.AST.BaseStmt): Declaration {
+    PythonHandler<Declaration, Python.AST.Def>(::ProblemDeclaration, frontend) {
+    override fun handleNode(node: Python.AST.Def): Declaration {
         return when (node) {
             is Python.AST.FunctionDef -> handleFunctionDef(node)
             is Python.AST.AsyncFunctionDef -> handleFunctionDef(node)
             is Python.AST.ClassDef -> handleClassDef(node)
-            else -> {
-                return handleNotSupported(node, node.javaClass.simpleName)
-            }
         }
     }
 
-    private fun handleNotSupported(node: Python.AST.BaseStmt, name: String): Declaration {
-        Util.errorWithFileLocation(
-            frontend,
-            node,
-            log,
-            "Parsing of type $name is not supported (yet)",
-        )
-
-        val cpgNode = this.configConstructor.get()
-        if (cpgNode is ProblemNode) {
-            cpgNode.problem = "Parsing of type $name is not supported (yet)"
-        }
-
-        return cpgNode
-    }
-
     /**
      * Translates a Python [`ClassDef`](https://docs.python.org/3/library/ast.html#ast.ClassDef)
      * into an [RecordDeclaration].
@@ -95,7 +75,7 @@ class DeclarationHandler(frontend: PythonLanguageFrontend) :
             when (s) {
                 // In order to be as compatible as possible with existing languages, we try to add
                 // declarations directly to the class
-                is Python.AST.WithDeclaration -> handle(s)
+                is Python.AST.Def -> handle(s)
                 // All other statements are added to the statements block of the class
                 else -> cls.statements += frontend.statementHandler.handle(s)
             }

diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt
@@ -26,6 +26,7 @@
 package de.fraunhofer.aisec.cpg.frontends.python
 
 import de.fraunhofer.aisec.cpg.graph.declarations.Declaration
+import de.fraunhofer.aisec.cpg.graph.statements.Statement
 import jep.python.PyObject
 
 /**
@@ -59,13 +60,20 @@ interface Python {
      */
     interface AST {
 
+        /**
+         * Represents a `ast.AST` node as returned by Python's `ast` parser.
+         *
+         * @param pyObject The Python object returned by jep.
+         */
+        interface AST {
+            var pyObject: PyObject
+        }
+
         /**
          * Some nodes, such as `ast.stmt` [AST.BaseStmt] and `ast.expr` [AST.BaseExpr] nodes have
          * extra location properties as implemented here.
          */
-        interface WithLocation { // TODO make the fields accessible `by lazy`
-            val pyObject: PyObject
-
+        interface WithLocation : AST { // TODO make the fields accessible `by lazy`
             /** Maps to the `lineno` filed from Python's ast. */
             val lineno: Int
                 get() {
@@ -92,21 +100,15 @@ interface Python {
         }
 
         /**
-         * Python does not really have "declarations", but rather these are also [AST.BaseStmt]s.
-         * But in order to be compatible with the remaining languages we need to ensure that
-         * elements such as functions or classes, still turn out to be [Declaration]s.
+         * Python does not really have "declarations", but it has "definitions". Instead of having
+         * their own AST class, they are also [AST.BaseStmt]s. In order to be compatible with the
+         * remaining languages we need to ensure that elements such as functions or classes, still
+         * turn out to be [Declaration]s, not [Statement]s
          *
          * This interface should be attached to all such statements that we consider to be
-         * declarations.
-         */
-        sealed interface WithDeclaration
-
-        /**
-         * Represents a `ast.AST` node as returned by Python's `ast` parser.
-         *
-         * @param pyObject The Python object returned by jep.
+         * definitions, and thus [Declaration]s.
          */
-        abstract class AST(pyObject: PyObject) : BaseObject(pyObject)
+        sealed interface Def : AST
 
         /**
          * ```
@@ -119,15 +121,15 @@ interface Python {
          *
          * Note: We currently only support `Module`s.
          */
-        abstract class BaseMod(pyObject: PyObject) : AST(pyObject)
+        abstract class BaseMod(pyObject: PyObject) : AST, BaseObject(pyObject)
 
         /**
          * ```
          * ast.Module = class Module(mod)
          *  |  Module(stmt* body, type_ignore* type_ignores)
          * ```
          */
-        class Module(pyObject: PyObject) : AST(pyObject) {
+        class Module(pyObject: PyObject) : AST, BaseObject(pyObject) {
             val body: kotlin.collections.List<BaseStmt> by lazy { "body" of pyObject }
 
             val type_ignores: kotlin.collections.List<type_ignore> by lazy {
@@ -167,7 +169,7 @@ interface Python {
          *  |  | Continue
          * ```
          */
-        sealed class BaseStmt(pyObject: PyObject) : AST(pyObject), WithLocation
+        sealed class BaseStmt(pyObject: PyObject) : AST, BaseObject(pyObject), WithLocation
 
         /**
          * Several classes are duplicated in the python AST for async and non-async variants. This
@@ -183,7 +185,7 @@ interface Python {
          * However, they are so similar, that we make use of this interface to avoid a lot of
          * duplicate code.
          */
-        sealed interface NormalOrAsyncFunctionDef : AsyncOrNot, WithDeclaration {
+        sealed interface NormalOrAsyncFunctionDef : AsyncOrNot, Def {
             val name: String
             val args: arguments
             val body: kotlin.collections.List<BaseStmt>
@@ -243,7 +245,7 @@ interface Python {
          *  |  ClassDef(identifier name, expr* bases, keyword* keywords, stmt* body, expr* decorator_list)
          * ```
          */
-        class ClassDef(pyObject: PyObject) : BaseStmt(pyObject), WithDeclaration {
+        class ClassDef(pyObject: PyObject) : BaseStmt(pyObject), Def {
             val name: String by lazy { "name" of pyObject }
 
             val bases: kotlin.collections.List<BaseExpr> by lazy { "bases" of pyObject }
@@ -564,7 +566,7 @@ interface Python {
          *
          * ast.expr = class expr(AST)
          */
-        sealed class BaseExpr(pyObject: PyObject) : AST(pyObject), WithLocation
+        sealed class BaseExpr(pyObject: PyObject) : AST, BaseObject(pyObject), WithLocation
 
         /**
          * ```
@@ -883,7 +885,7 @@ interface Python {
          *  |  boolop = And | Or
          * ```
          */
-        sealed class BaseBoolOp(pyObject: PyObject) : AST(pyObject)
+        sealed class BaseBoolOp(pyObject: PyObject) : AST, BaseObject(pyObject)
 
         /**
          * ```
@@ -906,7 +908,7 @@ interface Python {
          *  |  cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn
          * ```
          */
-        sealed class BaseCmpOp(pyObject: PyObject) : AST(pyObject)
+        sealed class BaseCmpOp(pyObject: PyObject) : AST, BaseObject(pyObject)
 
         /**
          * ```
@@ -994,7 +996,7 @@ interface Python {
          *  |  expr_context = Load | Store | Del
          * ```
          */
-        sealed class BaseExprContext(pyObject: PyObject) : AST(pyObject)
+        sealed class BaseExprContext(pyObject: PyObject) : AST, BaseObject(pyObject)
 
         /**
          * ```
@@ -1026,7 +1028,7 @@ interface Python {
          *  |  operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift | RShift | BitOr | BitXor | BitAnd | FloorDiv
          * ```
          */
-        sealed class BaseOperator(pyObject: PyObject) : AST(pyObject)
+        sealed class BaseOperator(pyObject: PyObject) : AST, BaseObject(pyObject)
 
         /**
          * ```
@@ -1145,7 +1147,7 @@ interface Python {
          *  |  | MatchOr(pattern* patterns)
          * ```
          */
-        abstract class BasePattern(pyObject: PyObject) : AST(pyObject), WithLocation
+        abstract class BasePattern(pyObject: PyObject) : AST, BaseObject(pyObject), WithLocation
 
         /**
          * ```
@@ -1246,7 +1248,7 @@ interface Python {
          *  |  unaryop = Invert | Not | UAdd | USub
          * ```
          */
-        sealed class BaseUnaryOp(pyObject: PyObject) : AST(pyObject)
+        sealed class BaseUnaryOp(pyObject: PyObject) : AST, BaseObject(pyObject)
 
         /**
          * ```
@@ -1286,7 +1288,7 @@ interface Python {
          *  |  alias(identifier name, identifier? asname)
          * ```
          */
-        class alias(pyObject: PyObject) : AST(pyObject), WithLocation {
+        class alias(pyObject: PyObject) : AST, BaseObject(pyObject), WithLocation {
             val name: String by lazy { "name" of pyObject }
             val asname: String? by lazy { "asname" of pyObject }
         }
@@ -1297,7 +1299,7 @@ interface Python {
          *  |  arg(identifier arg, expr? annotation, string? type_comment)
          * ```
          */
-        class arg(pyObject: PyObject) : AST(pyObject), WithLocation {
+        class arg(pyObject: PyObject) : AST, BaseObject(pyObject), WithLocation {
             val arg: String by lazy { "arg" of pyObject }
             val annotation: BaseExpr? by lazy { "annotation" of pyObject }
             val type_comment: String? by lazy { "type_comment" of pyObject }
@@ -1309,7 +1311,7 @@ interface Python {
          *  |  arguments(arg* posonlyargs, arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, arg? kwarg, expr* defaults)
          * ```
          */
-        class arguments(pyObject: PyObject) : AST(pyObject) {
+        class arguments(pyObject: PyObject) : AST, BaseObject(pyObject) {
             val posonlyargs: kotlin.collections.List<arg> by lazy { "posonlyargs" of pyObject }
             val args: kotlin.collections.List<arg> by lazy { "args" of pyObject }
             val vararg: arg? by lazy { "vararg" of pyObject }
@@ -1325,7 +1327,7 @@ interface Python {
          *  |  comprehension(expr target, expr iter, expr* ifs, int is_async)
          * ```
          */
-        class comprehension(pyObject: PyObject) : AST(pyObject) {
+        class comprehension(pyObject: PyObject) : AST, BaseObject(pyObject) {
             val target: BaseExpr by lazy { "target" of pyObject }
             val iter: BaseExpr by lazy { "iter" of pyObject }
             val ifs: kotlin.collections.List<BaseExpr> by lazy { "ifs" of pyObject }
@@ -1338,7 +1340,8 @@ interface Python {
          *  |  excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body)
          * ```
          */
-        sealed class BaseExcepthandler(python: PyObject) : AST(python), WithLocation
+        sealed class BaseExcepthandler(pyObject: PyObject) :
+            AST, BaseObject(pyObject), WithLocation
 
         /**
          * ast.ExceptHandler = class ExceptHandler(excepthandler) | ExceptHandler(expr? type,
@@ -1356,7 +1359,7 @@ interface Python {
          *  |  keyword(identifier? arg, expr value)
          * ```
          */
-        class keyword(pyObject: PyObject) : AST(pyObject), WithLocation {
+        class keyword(pyObject: PyObject) : AST, BaseObject(pyObject), WithLocation {
             val arg: String? by lazy { "arg" of pyObject }
             val value: BaseExpr by lazy { "value" of pyObject }
         }
@@ -1367,7 +1370,7 @@ interface Python {
          *  |  match_case(pattern pattern, expr? guard, stmt* body)
          * ```
          */
-        class match_case(pyObject: PyObject) : AST(pyObject) {
+        class match_case(pyObject: PyObject) : AST, BaseObject(pyObject) {
             val pattern: BasePattern by lazy { "pattern" of pyObject }
             val guard: BaseExpr? by lazy { "guard" of pyObject }
             val body: kotlin.collections.List<BaseStmt> by lazy { "body" of pyObject }
@@ -1381,15 +1384,15 @@ interface Python {
          *
          * TODO
          */
-        class type_ignore(pyObject: PyObject) : AST(pyObject)
+        class type_ignore(pyObject: PyObject) : AST, BaseObject(pyObject)
 
         /**
          * ```
          * ast.withitem = class withitem(AST)
          *  |  withitem(expr context_expr, expr? optional_vars)
          * ```
          */
-        class withitem(pyObject: PyObject) : AST(pyObject) {
+        class withitem(pyObject: PyObject) : AST, BaseObject(pyObject) {
             val context_expr: BaseExpr by lazy { "context_expr" of pyObject }
             val optional_vars: BaseExpr? by lazy { "optional_vars" of pyObject }
         }

diff --git a/...python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt b/...python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt
@@ -349,7 +349,7 @@ class PythonLanguageFrontend(language: Language<PythonLanguageFrontend>, ctx: Tr
                 when (stmt) {
                     // In order to be as compatible as possible with existing languages, we try to
                     // add declarations directly to the class
-                    is Python.AST.WithDeclaration -> declarationHandler.handle(stmt)
+                    is Python.AST.Def -> declarationHandler.handle(stmt)
                     // All other statements are added to the (static) statements block of the
                     // namespace.
                     else -> lastNamespace.statements += statementHandler.handle(stmt)