From 1de14ed54a1c88dd983473f00baafc15f28f0628 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Sun, 2 Feb 2025 22:59:44 +0100 Subject: [PATCH] Instead of caching importedSymbols statically once (or twice), instead we resolve the import declaration back to the namespace (declaration) and update the symbol on-chance. Adding an 'import' edge --- .../de/fraunhofer/aisec/cpg/ScopeManager.kt | 66 ++++++------- .../aisec/cpg/graph/DeclarationBuilder.kt | 5 +- .../fraunhofer/aisec/cpg/graph/Extensions.kt | 6 ++ .../graph/declarations/FunctionDeclaration.kt | 7 +- .../graph/declarations/ImportDeclaration.kt | 16 ++-- .../declarations/ParameterDeclaration.kt | 22 ++++- .../aisec/cpg/graph/edges/Extensions.kt | 15 +++ .../edges/collections/UnwrappedEdgeSet.kt | 13 +++ .../aisec/cpg/graph/edges/scopes/Import.kt | 96 +++++++++++++++++++ .../aisec/cpg/graph/scopes/NamespaceScope.kt | 39 +++++++- .../aisec/cpg/graph/scopes/Scope.kt | 28 +++++- .../graph/scopes/StructureDeclarationScope.kt | 6 +- .../cpg/graph/scopes/ValueDeclarationScope.kt | 7 +- .../aisec/cpg/passes/ImportResolver.kt | 80 ++++++++++++---- .../ResolveMemberExpressionAmbiguityPass.kt | 29 ++---- .../aisec/cpg/passes/TypeResolver.kt | 1 - .../aisec/cpg/passes/ImportResolverTest.kt | 13 ++- .../cpg/frontends/cxx/CXXLanguageFrontend.kt | 3 - .../cpg/frontends/cxx/DeclarationHandler.kt | 35 ++++--- .../frontends/golang/SpecificationHandler.kt | 9 +- .../aisec/cpg/passes/GoExtraPass.kt | 31 ------ .../frontends/java/JavaLanguageFrontend.kt | 6 +- .../cpg/frontends/python/StatementHandler.kt | 28 +++++- .../frontends/python/PythonFrontendTest.kt | 78 +++++++++++++-- .../src/test/resources/python/import_test.py | 1 + .../test/resources/python/import_vs_member.py | 23 ++++- 26 files changed, 500 insertions(+), 163 deletions(-) create mode 100644 cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/scopes/Import.kt diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt index 6c80422ad7e..492c140edd4 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt @@ -370,11 +370,11 @@ class ScopeManager : ScopeProvider { is ProblemDeclaration, is IncludeDeclaration -> { // directly add problems and includes to the global scope - this.globalScope?.addDeclaration(declaration, addToAST) + this.globalScope?.addDeclaration(declaration, addToAST, this) } is ValueDeclaration -> { val scope = this.firstScopeIsInstanceOrNull() - scope?.addDeclaration(declaration, addToAST) + scope?.addDeclaration(declaration, addToAST, this) } is ImportDeclaration, is EnumDeclaration, @@ -382,7 +382,7 @@ class ScopeManager : ScopeProvider { is NamespaceDeclaration, is TemplateDeclaration -> { val scope = this.firstScopeIsInstanceOrNull() - scope?.addDeclaration(declaration, addToAST) + scope?.addDeclaration(declaration, addToAST, this) } } } @@ -571,23 +571,25 @@ class ScopeManager : ScopeProvider { * the given [Name]. It also does this recursively. */ fun resolveParentAlias(name: Name, scope: Scope?): Name { - var parentName = name.parent ?: return name - parentName = resolveParentAlias(parentName, scope) + if (name.parent == null) { + return name + } - // Build a new name based on the eventual resolved parent alias - var newName = - if (parentName != name.parent) { - Name(name.localName, parentName, delimiter = name.delimiter) - } else { - name - } - var decl = - scope?.lookupSymbol(parentName.localName)?.singleOrNull { - it is NamespaceDeclaration || it is RecordDeclaration - } - if (decl != null && parentName != decl.name) { + val parentName = resolveParentAlias(name.parent, scope) + + // Look for an alias in the current scope. This is also resolves partial FQNs to their full + // FQN + var newScope = + scope + ?.lookupSymbol(parentName.localName) { + it is NamespaceDeclaration || it is RecordDeclaration + } + ?.map { nameScopeMap[it.name] } + ?.toSet() + ?.singleOrNull() + if (newScope != null) { // This is probably an already resolved alias so, we take this one - return Name(newName.localName, decl.name, delimiter = newName.delimiter) + return adjustNameIfNecessary(newScope.name, parentName, name) } // Some special handling of typedefs; this should somehow be merged with the above but not @@ -595,28 +597,26 @@ class ScopeManager : ScopeProvider { // but we rather want its original type name. // TODO: This really needs to be handled better somehow, maybe a common interface for // typedefs, namespaces and records that return the correct name? - decl = scope?.lookupSymbol(parentName.localName)?.singleOrNull { it is TypedefDeclaration } + val decl = + scope?.lookupSymbol(parentName.localName)?.singleOrNull { it is TypedefDeclaration } if ((decl as? TypedefDeclaration) != null) { - return Name(newName.localName, decl.type.name, delimiter = newName.delimiter) + return adjustNameIfNecessary(decl.type.name, parentName, name) } - // If we do not have a match yet, it could be that we are trying to resolve an FQN type - // during frontend translation. This is deprecated and will be replaced in the future - // by a system that also resolves type during symbol resolving. However, to support aliases - // from imports in this intermediate stage, we have to look for unresolved import - // declarations and also take their aliases into account - decl = - scope - ?.lookupSymbol(parentName.localName) - ?.filterIsInstance() - ?.singleOrNull() - if (decl != null && decl.importedSymbols.isEmpty() && parentName != decl.import) { - newName = Name(newName.localName, decl.import, delimiter = newName.delimiter) - } + // Otherwise, just build a new name based on the eventual resolved parent alias (if + // necessary) + var newName = adjustNameIfNecessary(parentName, name.parent, name) return newName } + private fun adjustNameIfNecessary(newParentName: Name, oldParentName: Name, name: Name): Name = + if (newParentName != oldParentName) { + Name(name.localName, newParentName, delimiter = name.delimiter) + } else { + name + } + /** * Directly jumps to a given scope. Returns the previous scope. Do not forget to set the scope * back to the old scope after performing the actions inside this scope. diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/DeclarationBuilder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/DeclarationBuilder.kt index 29a119aa2f7..1562c5bbb8b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/DeclarationBuilder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/DeclarationBuilder.kt @@ -30,6 +30,7 @@ import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend import de.fraunhofer.aisec.cpg.graph.Node.Companion.EMPTY_NAME import de.fraunhofer.aisec.cpg.graph.NodeBuilder.log import de.fraunhofer.aisec.cpg.graph.declarations.* +import de.fraunhofer.aisec.cpg.graph.edges.scopes.ImportStyle import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import de.fraunhofer.aisec.cpg.graph.statements.expressions.NewArrayExpression import de.fraunhofer.aisec.cpg.graph.types.Type @@ -449,7 +450,7 @@ fun MetadataProvider.newNamespaceDeclaration( @JvmOverloads fun MetadataProvider.newImportDeclaration( import: Name, - wildcardImport: Boolean = false, + style: ImportStyle, alias: Name? = null, rawNode: Any? = null, ): ImportDeclaration { @@ -457,7 +458,7 @@ fun MetadataProvider.newImportDeclaration( node.applyMetadata(this, "", rawNode) node.import = import node.alias = alias - node.wildcardImport = wildcardImport + node.style = style if (alias != null) { node.name = alias } else { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt index 7408d1c7180..6ec56969786 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt @@ -1360,6 +1360,12 @@ val Node.translationUnit: TranslationUnitDeclaration? return firstParentOrNull() } +/** Returns the [TranslationResult] where this node is located in. */ +val Node.translationResult: TranslationResult? + get() { + return firstParentOrNull() + } + /** Returns the [Component] where this node is located in. */ val Node.component: Component? get() { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt index ee3b9ae2687..cb494bcaf0a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt @@ -29,7 +29,6 @@ import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf -import de.fraunhofer.aisec.cpg.graph.edges.flows.Invoke import de.fraunhofer.aisec.cpg.graph.edges.flows.Invokes import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.edges.unwrappingIncoming @@ -73,10 +72,8 @@ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder, EOGStart Invokes(this, CallExpression::invokeEdges, outgoing = false) /** Virtual property for accessing [calledByEdges] without property edges. */ - val calledBy by - unwrappingIncoming( - FunctionDeclaration::calledByEdges - ) + val calledBy: MutableList by + unwrappingIncoming(FunctionDeclaration::calledByEdges) /** The list of return types. The default is an empty list. */ var returnTypes = listOf() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ImportDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ImportDeclaration.kt index c2d4009be29..977ad948b18 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ImportDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ImportDeclaration.kt @@ -27,6 +27,7 @@ package de.fraunhofer.aisec.cpg.graph.declarations import de.fraunhofer.aisec.cpg.PopulatedByPass import de.fraunhofer.aisec.cpg.graph.Name +import de.fraunhofer.aisec.cpg.graph.edges.scopes.ImportStyle import de.fraunhofer.aisec.cpg.graph.scopes.FileScope import de.fraunhofer.aisec.cpg.graph.scopes.NameScope import de.fraunhofer.aisec.cpg.graph.scopes.Scope @@ -43,7 +44,7 @@ import org.neo4j.ogm.annotation.typeconversion.Convert * * ### Examples (Go) * - * In Go, we usually import the package itself as a symbol. + * In Go, we usually import the package itself as a symbol (see [ImportStyle.IMPORT_NAMESPACE]). * * ```Go * package p @@ -101,7 +102,8 @@ import org.neo4j.ogm.annotation.typeconversion.Convert * ``` * * The imported symbol is then visible within the current [Scope] of the [ImportDeclaration]. In the - * example [name] and [import] is set to `std::string`, [wildcardImport] is `false`. + * example [name] and [import] is set to `std::string`, [style] is + * [ImportStyle.IMPORT_SINGLE_SYMBOL_FROM_NAMESPACE]. * * Another possibility is to import a complete namespace, or to be more precise import all symbols * of the specified namespace into the current scope. @@ -118,7 +120,8 @@ import org.neo4j.ogm.annotation.typeconversion.Convert * } * ``` * - * In this example, the [name] and [import] is set to `std` and [wildcardImport] is `true`. + * In this example, the [name] and [import] is set to `std` and [style] is + * [ImportStyle.IMPORT_ALL_SYMBOLS_FROM_NAMESPACE]. */ class ImportDeclaration : Declaration() { @@ -149,11 +152,8 @@ class ImportDeclaration : Declaration() { */ var importURL: String? = null - /** - * Specifies that [name] is pointing to a [NameScope] and that all [Scope.symbols] of that name - * scope need to be imported into the scope this declaration lives in. - */ - var wildcardImport: Boolean = false + /** The import style. */ + var style: ImportStyle = ImportStyle.IMPORT_SINGLE_SYMBOL_FROM_NAMESPACE /** * A list of symbols that this declaration imports. This will be populated by diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ParameterDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ParameterDeclaration.kt index cd820498d05..5ac13b88580 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ParameterDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ParameterDeclaration.kt @@ -25,6 +25,7 @@ */ package de.fraunhofer.aisec.cpg.graph.declarations +import de.fraunhofer.aisec.cpg.graph.ArgumentHolder import de.fraunhofer.aisec.cpg.graph.HasDefault import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf import de.fraunhofer.aisec.cpg.graph.edges.unwrapping @@ -33,7 +34,7 @@ import java.util.* import org.neo4j.ogm.annotation.Relationship /** A declaration of a function or nontype template parameter. */ -class ParameterDeclaration : ValueDeclaration(), HasDefault { +class ParameterDeclaration : ValueDeclaration(), HasDefault, ArgumentHolder { var isVariadic = false @Relationship(value = "DEFAULT", direction = Relationship.Direction.OUTGOING) @@ -57,4 +58,23 @@ class ParameterDeclaration : ValueDeclaration(), HasDefault { } override fun hashCode() = Objects.hash(super.hashCode(), isVariadic, defaultValue) + + override fun addArgument(expression: Expression) { + if (defaultValue == null) { + defaultValue = expression + } + } + + override fun replaceArgument(old: Expression, new: Expression): Boolean { + if (defaultValue == old) { + defaultValue = new + return true + } + + return false + } + + override fun hasArgument(expression: Expression): Boolean { + return defaultValue == expression + } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Extensions.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Extensions.kt index 165217d4b73..609f2af80dc 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Extensions.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Extensions.kt @@ -71,6 +71,21 @@ fun < return edge.unwrap().IncomingDelegate() } +/** See [UnwrappedEdgeSet.IncomingDelegate]. */ +fun < + IncomingType : Node, + PropertyType : Node, + NodeType : Node, + EdgeType : Edge, +> NodeType.unwrappingIncoming( + edgeProperty: KProperty1> +): UnwrappedEdgeSet.IncomingDelegate { + // Create an unwrapped container out of the edge property... + edgeProperty.isAccessible = true + val edge = edgeProperty.call(this) + return edge.unwrap().IncomingDelegate() +} + /** See [UnwrappedEdgeSet.Delegate]. */ fun > NodeType.unwrapping( edgeProperty: KProperty1> diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeSet.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeSet.kt index 6f6301ac89d..0d45af6c986 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeSet.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeSet.kt @@ -59,6 +59,19 @@ class UnwrappedEdgeSet>( } } + /** See [UnwrappedEdgeList.IncomingDelegate], but as a [MutableSet] instead of [MutableList]. */ + @Transient + inner class IncomingDelegate() { + operator fun getValue(thisRef: ThisType, property: KProperty<*>): MutableSet { + @Suppress("UNCHECKED_CAST") + return this@UnwrappedEdgeSet as MutableSet + } + + operator fun setValue(thisRef: ThisType, property: KProperty<*>, value: Set) { + @Suppress("UNCHECKED_CAST") this@UnwrappedEdgeSet.resetTo(value as Collection) + } + } + operator fun provideDelegate( thisRef: ThisType, prop: KProperty<*>, diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/scopes/Import.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/scopes/Import.kt new file mode 100644 index 00000000000..51082c1fa66 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/scopes/Import.kt @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2025, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.scopes + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.declarations.ImportDeclaration +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeSet +import de.fraunhofer.aisec.cpg.graph.edges.collections.MirroredEdgeCollection +import de.fraunhofer.aisec.cpg.graph.scopes.NamespaceScope +import de.fraunhofer.aisec.cpg.graph.scopes.Scope +import kotlin.reflect.KProperty + +/** + * The style of the import. This can be used to distinguish between different import modes, such as + * importing a single symbol from a namespace, importing a whole namespace or importing all symbols + * from a namespace. + */ +enum class ImportStyle { + /** + * Imports a single symbol from the target namespace. The current scope will contain a symbol + * with the same name as the imported symbol. + * + * Note: Some languages support importing more than one symbol at a time. In this case, the list + * is split into multiple [ImportDeclaration] nodes (and [Import] edges). + */ + IMPORT_SINGLE_SYMBOL_FROM_NAMESPACE, + + /** + * Imports the target namespace as a single symbol. The current scope will contain a symbol with + * the same name as the imported namespace. + */ + IMPORT_NAMESPACE, + + /** + * Imports all symbols from the target namespace. The current scope will contain one new symbol + * for each symbol in the namespace. + * + * This is also known as a "wildcard" import. + */ + IMPORT_ALL_SYMBOLS_FROM_NAMESPACE, +} + +/** + * This edge represents the import of a [NamespaceScope] into another [Scope]. The [style] of import + * (e.g., whether only a certain symbol or the whole namespace is imported) is determined by the + * [declaration]. + */ +class Import(start: Scope, end: NamespaceScope, var declaration: ImportDeclaration? = null) : + Edge(start, end) { + + override var labels = setOf("SCOPE_IMPORT") + + val style: ImportStyle? + get() = declaration?.style + + init { + declaration?.import?.let { this.name = it.localName } + } +} + +/** A container to manage [Import] edges. */ +class Imports( + thisRef: Node, + override var mirrorProperty: KProperty>, + outgoing: Boolean = true, +) : + EdgeSet( + thisRef, + init = { start, end -> Import(start as Scope, end) }, + outgoing = outgoing, + ), + MirroredEdgeCollection diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/NamespaceScope.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/NamespaceScope.kt index 01c6266c009..241283d56fd 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/NamespaceScope.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/NamespaceScope.kt @@ -25,11 +25,48 @@ */ package de.fraunhofer.aisec.cpg.graph.scopes +import de.fraunhofer.aisec.cpg.ScopeManager +import de.fraunhofer.aisec.cpg.graph.declarations.Declaration +import de.fraunhofer.aisec.cpg.graph.declarations.ImportDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.NamespaceDeclaration +import de.fraunhofer.aisec.cpg.graph.edges.scopes.Imports +import de.fraunhofer.aisec.cpg.graph.edges.unwrappingIncoming +import de.fraunhofer.aisec.cpg.passes.updateImportedSymbols +import org.neo4j.ogm.annotation.Relationship /** * This scope is opened up by a [NamespaceDeclaration] and represents the scope of the whole * namespace. This scope is special in a way that it will only exist once (per [GlobalScope]) and * contains all symbols declared in this namespace, even if they are spread across multiple files. */ -class NamespaceScope(astNode: NamespaceDeclaration) : NameScope(astNode) +class NamespaceScope(astNode: NamespaceDeclaration) : NameScope(astNode) { + + /** + * This is the mirror property to [Scope.importedScopeEdges]. It specifies which other [Scope]s + * are importing this namespace. + * + * This is used in [addDeclaration] to update the [ImportDeclaration.importedSymbols] once we + * add a new symbol here, so that is it also visible in the scope of the [ImportDeclaration]. + */ + @Relationship(value = "IMPORTS_SCOPE", direction = Relationship.Direction.INCOMING) + val importedByEdges: Imports = + Imports(this, mirrorProperty = Scope::importedScopeEdges, outgoing = false) + + /** Virtual property for accessing [importedScopeEdges] without property edges. */ + val importedBy: MutableSet by unwrappingIncoming(NamespaceScope::importedByEdges) + + override fun addDeclaration( + declaration: Declaration, + addToAST: Boolean, + scopeManager: ScopeManager, + ) { + val result = super.addDeclaration(declaration, addToAST, scopeManager) + + // Update imported symbols of dependent scopes + for (edge in importedByEdges) { + edge.declaration?.let { scopeManager.updateImportedSymbols(it) } + } + + return result + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt index 325e0bbb39a..ff29cdf3a10 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt @@ -26,15 +26,21 @@ package de.fraunhofer.aisec.cpg.graph.scopes import com.fasterxml.jackson.annotation.JsonBackReference +import de.fraunhofer.aisec.cpg.PopulatedByPass import de.fraunhofer.aisec.cpg.frontends.HasImplicitReceiver import de.fraunhofer.aisec.cpg.frontends.Language import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.ImportDeclaration +import de.fraunhofer.aisec.cpg.graph.edges.scopes.Import +import de.fraunhofer.aisec.cpg.graph.edges.scopes.ImportStyle +import de.fraunhofer.aisec.cpg.graph.edges.scopes.Imports +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.firstScopeParentOrNull import de.fraunhofer.aisec.cpg.graph.statements.LabelStatement import de.fraunhofer.aisec.cpg.graph.statements.LookupScopeStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference +import de.fraunhofer.aisec.cpg.passes.ImportResolver import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.NodeEntity import org.neo4j.ogm.annotation.Relationship @@ -79,10 +85,25 @@ sealed class Scope( @Transient var symbols: SymbolMap = mutableMapOf() /** - * A list of [ImportDeclaration] nodes that have [ImportDeclaration.wildcardImport] set to true. + * A list of [ImportDeclaration] nodes that have an + * [ImportStyle.IMPORT_ALL_SYMBOLS_FROM_NAMESPACE] import style ("wildcard" import). */ @Transient var wildcardImports: MutableSet = mutableSetOf() + /** + * This set of edges is used to store [Import] edges that denotes foreign [NamespaceScope] + * information that is imported into this scope. The edge holds information about the "style" of + * the import (see [ImportStyle]) and the [ImportDeclaration] that is responsible for this. The + * property is populated by the [ImportResolver]. + */ + @Relationship(value = "IMPORTS_SCOPE", direction = Relationship.Direction.OUTGOING) + @PopulatedByPass(ImportResolver::class) + val importedScopeEdges = + Imports(this, mirrorProperty = NamespaceScope::importedByEdges, outgoing = true) + + /** Virtual property for accessing [importedScopeEdges] without property edges. */ + val importedScopes by unwrapping(Scope::importedScopeEdges) + /** * In some languages, the lookup scope of a symbol that is being resolved (e.g. of a * [Reference]) can be adjusted through keywords (such as `global` in Python or PHP). @@ -95,7 +116,10 @@ sealed class Scope( /** Adds a [declaration] with the defined [symbol]. */ fun addSymbol(symbol: Symbol, declaration: Declaration) { - if (declaration is ImportDeclaration && declaration.wildcardImport) { + if ( + declaration is ImportDeclaration && + declaration.style == ImportStyle.IMPORT_ALL_SYMBOLS_FROM_NAMESPACE + ) { // Because a wildcard import does not really have a valid "symbol", we store it in a // separate list wildcardImports += declaration diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/StructureDeclarationScope.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/StructureDeclarationScope.kt index 76a2795f680..8078a72bb47 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/StructureDeclarationScope.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/StructureDeclarationScope.kt @@ -60,7 +60,11 @@ sealed class StructureDeclarationScope(astNode: Node?) : ValueDeclarationScope(a } } - override fun addDeclaration(declaration: Declaration, addToAST: Boolean) { + override fun addDeclaration( + declaration: Declaration, + addToAST: Boolean, + scopeManager: ScopeManager, + ) { if (declaration is ValueDeclaration) { addValueDeclaration(declaration, addToAST) } else { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/ValueDeclarationScope.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/ValueDeclarationScope.kt index 2e3e81372b9..12c23cd50b0 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/ValueDeclarationScope.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/ValueDeclarationScope.kt @@ -25,6 +25,7 @@ */ package de.fraunhofer.aisec.cpg.graph.scopes +import de.fraunhofer.aisec.cpg.ScopeManager import de.fraunhofer.aisec.cpg.graph.DeclarationHolder import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node @@ -55,7 +56,11 @@ sealed class ValueDeclarationScope(astNode: Node?) : Scope(astNode) { typedefs[typedef.alias.name] = typedef } - open fun addDeclaration(declaration: Declaration, addToAST: Boolean) { + open fun addDeclaration( + declaration: Declaration, + addToAST: Boolean, + scopeManager: ScopeManager, + ) { if (declaration is ValueDeclaration) { addValueDeclaration(declaration, addToAST) } else { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ImportResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ImportResolver.kt index 3c427d0da5c..3500b939dd6 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ImportResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ImportResolver.kt @@ -25,6 +25,7 @@ */ package de.fraunhofer.aisec.cpg.passes +import de.fraunhofer.aisec.cpg.ScopeManager import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.graph.Component @@ -34,8 +35,10 @@ import de.fraunhofer.aisec.cpg.graph.component import de.fraunhofer.aisec.cpg.graph.declarations.ImportDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.NamespaceDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration -import de.fraunhofer.aisec.cpg.graph.namespaces +import de.fraunhofer.aisec.cpg.graph.edges.scopes.Import +import de.fraunhofer.aisec.cpg.graph.edges.scopes.ImportStyle import de.fraunhofer.aisec.cpg.graph.scopes.NameScope +import de.fraunhofer.aisec.cpg.graph.scopes.NamespaceScope import de.fraunhofer.aisec.cpg.graph.scopes.Scope import de.fraunhofer.aisec.cpg.graph.translationUnit import de.fraunhofer.aisec.cpg.helpers.IdentitySet @@ -44,9 +47,8 @@ import de.fraunhofer.aisec.cpg.helpers.Util.errorWithFileLocation import de.fraunhofer.aisec.cpg.helpers.identitySetOf import de.fraunhofer.aisec.cpg.helpers.toIdentitySet import de.fraunhofer.aisec.cpg.passes.Pass.Companion.log +import de.fraunhofer.aisec.cpg.passes.inference.tryNamespaceInference import de.fraunhofer.aisec.cpg.processing.strategy.Strategy -import de.fraunhofer.aisec.cpg.processing.strategy.Strategy.COMPONENTS_LEAST_IMPORTS -import de.fraunhofer.aisec.cpg.processing.strategy.Strategy.TRANSLATION_UNITS_LEAST_IMPORTS import java.util.IdentityHashMap /** @@ -229,6 +231,9 @@ class ImportResolver(ctx: TranslationContext) : TranslationResultPass(ctx) { return } + // Populate the imported scopes edges + import.populateImportedScopes() + // Let's look for imported namespaces // First, we collect the individual parts of the name var parts = mutableListOf() @@ -274,12 +279,6 @@ class ImportResolver(ctx: TranslationContext) : TranslationResultPass(ctx) { // file. We only want to depend on the particular translation unit that is the // authoritative source of this namespace and this is the case if there is no // sub-declaration. - /*namespaces = - namespaces.filter { - // Note: the "namespaces" extension contains the starting node itself as well, - // so if we have no sub-namespace declaration, the size == 1 - it.namespaces.size == 1 - }*/ // Next, we loop through all namespaces in order to "connect" them to our current module for (declaration in namespaces) { @@ -325,7 +324,50 @@ class ImportResolver(ctx: TranslationContext) : TranslationResultPass(ctx) { } private fun handleImportDeclaration(import: ImportDeclaration) { - import.updateImportedSymbols() + ctx.scopeManager.updateImportedSymbols(import) + } + + /** + * This function populates the [Scope.importedScopes] property of the [Scope] that the + * [ImportDeclaration] "lives" in. + */ + private fun ImportDeclaration.populateImportedScopes() { + val startScope = scope + val name = + when (style) { + ImportStyle.IMPORT_SINGLE_SYMBOL_FROM_NAMESPACE -> { + import.parent + } + + ImportStyle.IMPORT_ALL_SYMBOLS_FROM_NAMESPACE, + ImportStyle.IMPORT_NAMESPACE -> { + import + } + } + if (name == null) { + errorWithFileLocation(this, log, "Could not get namespace name from import declaration") + return + } else if (startScope == null) { + errorWithFileLocation(this, log, "Could not get scope from import declaration") + return + } + + // Try to look up the namespace scope by the name + var targetScope = scopeManager.lookupScope(name) as? NamespaceScope + if (targetScope == null) { + // Try to infer it, if inference is configured + val decl = tryNamespaceInference(name, this) + if (decl != null) { + targetScope = scopeManager.lookupScope(name) as? NamespaceScope + } + } + + // If we have a target scope, we can create an "import" edge + if (targetScope != null) { + // Create a new import edge with all the necessary information + val edge = Import(startScope, targetScope, this) + startScope.importedScopeEdges += edge + } } override fun cleanup() { @@ -339,29 +381,27 @@ class ImportResolver(ctx: TranslationContext) : TranslationResultPass(ctx) { * namespaces that are imported at a later stage (e.g., in the [TypeResolver]), otherwise they won't * be visible to the later passes. */ -context(Pass<*>) -fun ImportDeclaration.updateImportedSymbols() { +fun ScopeManager.updateImportedSymbols(import: ImportDeclaration) { // We always need to search at the global scope because we are "importing" something, so by // definition, this is not in the scope of the current file. - val scope = scopeManager.globalScope ?: return + val scope = globalScope ?: return // Let's do some importing. We need to import either a wildcard - if (this.wildcardImport) { - val list = scopeManager.lookupSymbolByName(this.import, this.language, this.location, scope) + if (import.style == ImportStyle.IMPORT_ALL_SYMBOLS_FROM_NAMESPACE) { + val list = lookupSymbolByName(import.import, import.language, import.location, scope) val symbol = list.singleOrNull() if (symbol != null) { // In this case, the symbol must point to a name scope - val symbolScope = scopeManager.lookupScope(symbol) + val symbolScope = lookupScope(symbol) if (symbolScope is NameScope) { - this.importedSymbols = symbolScope.symbols + import.importedSymbols = symbolScope.symbols } } } else { // or a symbol directly val list = - scopeManager - .lookupSymbolByName(this.import, this.language, this.location, scope) + lookupSymbolByName(import.import, import.language, import.location, scope) .toMutableList() - this.importedSymbols = mutableMapOf(this.symbol to list) + import.importedSymbols = mutableMapOf(import.symbol to list) } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ResolveMemberExpressionAmbiguityPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ResolveMemberExpressionAmbiguityPass.kt index 44b1b195ded..264d04ee157 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ResolveMemberExpressionAmbiguityPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ResolveMemberExpressionAmbiguityPass.kt @@ -31,15 +31,12 @@ import de.fraunhofer.aisec.cpg.frontends.HasMemberExpressionAmbiguity import de.fraunhofer.aisec.cpg.graph.HasBase import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.codeAndLocationFrom -import de.fraunhofer.aisec.cpg.graph.declarations.ImportDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.NamespaceDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.fqn -import de.fraunhofer.aisec.cpg.graph.imports import de.fraunhofer.aisec.cpg.graph.newReference import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression -import de.fraunhofer.aisec.cpg.graph.translationUnit import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker import de.fraunhofer.aisec.cpg.helpers.replace import de.fraunhofer.aisec.cpg.passes.configuration.DependsOn @@ -103,12 +100,8 @@ class ResolveMemberExpressionAmbiguityPass(ctx: TranslationContext) : Translatio * function returns the fully qualified name of the import. If the name does not refer to an * import, returns null. * - * The check happens in two stages: - * - First, the function looks up the name in the current scope. If a symbol is found that - * represents a [NamespaceDeclaration], the name of the declaration is returned. - * - Second, if no symbol is found in the current scope, the function looks up the name in the - * list of [ImportDeclaration]s of the translation unit. If an import is found that matches - * the name, the name of the import is returned. + * The function looks up the name in the current scope. If a symbol is found that represents a + * [NamespaceDeclaration], the name of the declaration is returned. * * @param name The name to check for an import. * @param hint The expression that hints at the language and location. @@ -121,20 +114,12 @@ class ResolveMemberExpressionAmbiguityPass(ctx: TranslationContext) : Translatio language = hint.language, location = hint.location, startScope = hint.scope, + predicate = { it is NamespaceDeclaration }, ) - var declaration = resolved.singleOrNull() - var isImportedNamespace = declaration is NamespaceDeclaration - if (!isImportedNamespace) { - // It still could be an imported namespace of an imported package that we do not know. - // The problem is that we do not really know at this point whether we import a - // (sub)module or a global variable of the namespace. We tend to assume that this is a - // namespace - val imports = hint.translationUnit.imports - declaration = - imports.firstOrNull { it.name.lastPartsMatch(name) || it.name.startsWith(name) } - isImportedNamespace = declaration != null - } - + // There can be multiple declarations for the same namespace because the declaration can + // exist multiple times, but per definition in the scope manager, they all point to the same + // namespace, so we can just pick the first one. + var declaration = resolved.firstOrNull() return declaration?.name } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt index 99c7fbeeddd..a3db0c96c9b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt @@ -57,7 +57,6 @@ open class TypeResolver(ctx: TranslationContext) : ComponentPass(ctx) { TypeResolver::class.java, "Updating imported symbols for ${component.imports.size} imports", ) - component.imports.forEach { it.updateImportedSymbols() } b.stop() } diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/ImportResolverTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/ImportResolverTest.kt index ccdd813eb28..5036c7460a1 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/ImportResolverTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/ImportResolverTest.kt @@ -32,6 +32,7 @@ import de.fraunhofer.aisec.cpg.TypeManager import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.builder.translationResult +import de.fraunhofer.aisec.cpg.graph.edges.scopes.ImportStyle import de.fraunhofer.aisec.cpg.graph.newImportDeclaration import de.fraunhofer.aisec.cpg.graph.newNamespaceDeclaration import de.fraunhofer.aisec.cpg.graph.newTranslationUnitDeclaration @@ -67,9 +68,17 @@ class ImportResolverTest { var pkgB = newNamespaceDeclaration("b") scopeManager.addDeclaration(pkgB) scopeManager.enterScope(pkgB) - var import = newImportDeclaration(parseName("a")) + var import = + newImportDeclaration( + parseName("a"), + style = ImportStyle.IMPORT_NAMESPACE, + ) scopeManager.addDeclaration(import) - import = newImportDeclaration(parseName("c.bar")) + import = + newImportDeclaration( + parseName("c.bar"), + style = ImportStyle.IMPORT_SINGLE_SYMBOL_FROM_NAMESPACE, + ) scopeManager.addDeclaration(import) scopeManager.leaveScope(pkgB) tuB diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontend.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontend.kt index ca508c5f97a..f10ac733d46 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontend.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontend.kt @@ -690,9 +690,6 @@ open class CXXLanguageFrontend(language: Language, ctx: Tra parseName(name.toString()) } - // We need to take name(space) aliases into account. - typeName = scopeManager.resolveParentAlias(typeName, scopeManager.currentScope) - return objectType(typeName, rawNode = name) } diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclarationHandler.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclarationHandler.kt index e058ff27ebf..2da0c7b3b10 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclarationHandler.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclarationHandler.kt @@ -27,6 +27,7 @@ package de.fraunhofer.aisec.cpg.frontends.cxx import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* +import de.fraunhofer.aisec.cpg.graph.edges.scopes.ImportStyle import de.fraunhofer.aisec.cpg.graph.scopes.NameScope import de.fraunhofer.aisec.cpg.graph.scopes.RecordScope import de.fraunhofer.aisec.cpg.graph.scopes.ValueDeclarationScope @@ -81,15 +82,16 @@ class DeclarationHandler(lang: CXXLanguageFrontend) : } /** - * Translates a C++ (namespace - * alias)[https://en.cppreference.com/w/cpp/language/namespace_alias] into an alias handled by - * an [ImportDeclaration]. + * Translates a C++ + * [namespace alias](https://en.cppreference.com/w/cpp/language/namespace_alias) into an alias + * handled by an [ImportDeclaration]. */ private fun handleNamespaceAlias(ctx: CPPASTNamespaceAlias): ImportDeclaration { val from = parseName(ctx.mappingName.toString()) val to = parseName(ctx.alias.toString()) - val import = newImportDeclaration(from, false, to, rawNode = ctx) + val import = + newImportDeclaration(from, style = ImportStyle.IMPORT_NAMESPACE, to, rawNode = ctx) frontend.scopeManager.addDeclaration(import) @@ -97,14 +99,18 @@ class DeclarationHandler(lang: CXXLanguageFrontend) : } /** - * Translates a C++ (using - * directive)[https://en.cppreference.com/w/cpp/language/namespace#Using-directives] into a - * [ImportDeclaration]. + * Translates a C++ + * [using directive](https://en.cppreference.com/w/cpp/language/namespace#Using-directives) into + * a [ImportDeclaration]. */ private fun handleUsingDirective(ctx: CPPASTUsingDirective): Declaration { val import = parseName(ctx.qualifiedName.toString()) - val declaration = newImportDeclaration(import, rawNode = ctx) - declaration.wildcardImport = true + val declaration = + newImportDeclaration( + import, + style = ImportStyle.IMPORT_ALL_SYMBOLS_FROM_NAMESPACE, + rawNode = ctx, + ) frontend.scopeManager.addDeclaration(declaration) @@ -112,13 +118,18 @@ class DeclarationHandler(lang: CXXLanguageFrontend) : } /** - * Translates a C++ (using - * declaration)[https://en.cppreference.com/w/cpp/language/using_declaration] into a + * Translates a C++ + * [using declaration](https://en.cppreference.com/w/cpp/language/using_declaration) into a * [ImportDeclaration]. */ private fun handleUsingDeclaration(ctx: CPPASTUsingDeclaration): Declaration { val import = parseName(ctx.name.toString()) - val declaration = newImportDeclaration(import, rawNode = ctx) + val declaration = + newImportDeclaration( + import, + style = ImportStyle.IMPORT_SINGLE_SYMBOL_FROM_NAMESPACE, + rawNode = ctx, + ) frontend.scopeManager.addDeclaration(declaration) diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/SpecificationHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/SpecificationHandler.kt index d92ab3bef37..63bf4e6b851 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/SpecificationHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/SpecificationHandler.kt @@ -27,6 +27,7 @@ package de.fraunhofer.aisec.cpg.frontends.golang import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* +import de.fraunhofer.aisec.cpg.graph.edges.scopes.ImportStyle import de.fraunhofer.aisec.cpg.graph.scopes.NameScope import de.fraunhofer.aisec.cpg.helpers.Util @@ -63,7 +64,13 @@ class SpecificationHandler(frontend: GoLanguageFrontend) : } } - val import = newImportDeclaration(import = name, alias = alias, rawNode = importSpec) + val import = + newImportDeclaration( + import = name, + alias = alias, + style = ImportStyle.IMPORT_NAMESPACE, + rawNode = importSpec, + ) import.importURL = filename return import diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt index 2d7f4e388bd..3f463506a40 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt @@ -37,7 +37,6 @@ import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker import de.fraunhofer.aisec.cpg.passes.configuration.DependsOn import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteBefore -import de.fraunhofer.aisec.cpg.passes.inference.startInference /** * This pass takes care of several things that we need to clean up, once all translation units are @@ -113,7 +112,6 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx) { walker = SubgraphWalker.ScopedWalker(scopeManager) walker.registerHandler { _, _, node -> when (node) { - is ImportDeclaration -> handleImportDeclaration(node) is RecordDeclaration -> handleRecordDeclaration(node) is AssignExpression -> handleAssign(node) is ForEachStatement -> handleForEachStatement(node) @@ -375,35 +373,6 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx) { } } - /** - * This function gets called for every [IncludeDeclaration] (which in Go imports a whole - * package) and checks, if we need to infer a [NamespaceDeclaration] for this particular - * include. - */ - // TODO: Somehow, this gets called twice?! - private fun handleImportDeclaration(import: ImportDeclaration) { - // If the namespace is included as _, we can ignore it, as its only included as a runtime - // dependency - if (import.name.localName == "_") { - return - } - - // Try to see if we already know about this namespace somehow - val namespace = - scopeManager.lookupSymbolByNodeName(import) { - it is NamespaceDeclaration && it.path == import.importURL - } - - // If not, we can infer a namespace declaration, so we can bundle all inferred function - // declarations in there - if (namespace.isEmpty()) { - scopeManager.globalScope - ?.astNode - ?.startInference(ctx) - ?.inferNamespaceDeclaration(import.name, import.importURL, import) - } - } - override fun cleanup() { // Nothing to do } diff --git a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontend.kt b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontend.kt index b14f987f1d5..0fd9d65856f 100644 --- a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontend.kt +++ b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontend.kt @@ -58,6 +58,7 @@ import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.Annotation import de.fraunhofer.aisec.cpg.graph.declarations.NamespaceDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration +import de.fraunhofer.aisec.cpg.graph.edges.scopes.ImportStyle import de.fraunhofer.aisec.cpg.graph.scopes.Scope import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import de.fraunhofer.aisec.cpg.helpers.Benchmark @@ -145,7 +146,10 @@ open class JavaLanguageFrontend(language: Language, ctx: T // We create an implicit import for "java.lang.*" val decl = - newImportDeclaration(parseName("java.lang"), wildcardImport = true) + newImportDeclaration( + parseName("java.lang"), + style = ImportStyle.IMPORT_ALL_SYMBOLS_FROM_NAMESPACE, + ) .implicit("import java.lang.*") scopeManager.addDeclaration(decl) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index 1b4640926b3..ccd3062371d 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -33,6 +33,7 @@ import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage.Companion.MODIFIE import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.Annotation import de.fraunhofer.aisec.cpg.graph.declarations.* +import de.fraunhofer.aisec.cpg.graph.edges.scopes.ImportStyle import de.fraunhofer.aisec.cpg.graph.scopes.FunctionScope import de.fraunhofer.aisec.cpg.graph.scopes.NameScope import de.fraunhofer.aisec.cpg.graph.scopes.NamespaceScope @@ -542,12 +543,16 @@ class StatementHandler(frontend: PythonLanguageFrontend) : if (alias != null) { newImportDeclaration( parseName(imp.name), - false, + style = ImportStyle.IMPORT_NAMESPACE, parseName(alias), rawNode = imp, ) } else { - newImportDeclaration(parseName(imp.name), false, rawNode = imp) + newImportDeclaration( + parseName(imp.name), + style = ImportStyle.IMPORT_NAMESPACE, + rawNode = imp, + ) } frontend.scopeManager.addDeclaration(decl) declStmt.declarationEdges += decl @@ -595,16 +600,29 @@ class StatementHandler(frontend: PythonLanguageFrontend) : if (imp.name == "*") { // In the wildcard case, our "import" is the module name, and we set "wildcard" // to true - newImportDeclaration(module, true, rawNode = imp) + newImportDeclaration( + module, + style = ImportStyle.IMPORT_ALL_SYMBOLS_FROM_NAMESPACE, + rawNode = imp, + ) } else { // If we import an individual symbol, we need to FQN the symbol with our module // name and import that. We also need to take care of any alias val name = module.fqn(imp.name) val alias = imp.asname if (alias != null) { - newImportDeclaration(name, false, parseName(alias), rawNode = imp) + newImportDeclaration( + name, + style = ImportStyle.IMPORT_SINGLE_SYMBOL_FROM_NAMESPACE, + parseName(alias), + rawNode = imp, + ) } else { - newImportDeclaration(name, false, rawNode = imp) + newImportDeclaration( + name, + style = ImportStyle.IMPORT_SINGLE_SYMBOL_FROM_NAMESPACE, + rawNode = imp, + ) } } diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index 491a52faa9c..24a1887e5ae 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -35,6 +35,7 @@ import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.ParameterDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.edges.* +import de.fraunhofer.aisec.cpg.graph.edges.scopes.ImportStyle import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.ListType @@ -1416,6 +1417,21 @@ class PythonFrontendTest : BaseTest() { } assertNotNull(result) + // import a + val importA = result.imports["a"] + assertNotNull(importA) + assertEquals(ImportStyle.IMPORT_NAMESPACE, importA.style) + assertContains( + assertNotNull(importA.scope?.importedScopes), + assertNotNull(result.finalCtx.scopeManager.lookupScope(Name("a"))), + ) + + // from c import * + val importC = result.imports["c"] + assertNotNull(importC) + assertEquals(ImportStyle.IMPORT_ALL_SYMBOLS_FROM_NAMESPACE, importC.style) + // assertEquals(result.namespaces["c"], importC.importedFrom) + val aFunc = result.functions["a.func"] assertNotNull(aFunc) @@ -1638,18 +1654,62 @@ class PythonFrontendTest : BaseTest() { @Test fun testImportVsMember() { val topLevel = Path.of("src", "test", "resources", "python") - val tu = - analyzeAndGetFirstTU( - listOf(topLevel.resolve("import_vs_member.py").toFile()), - topLevel, - true, - ) { + val result = + analyze(listOf(topLevel.resolve("import_vs_member.py").toFile()), topLevel, true) { it.registerLanguage() } - assertNotNull(tu) + assertNotNull(result) - val refs = tu.refs - refs.forEach { assertIsNot(it) } + val pkg = result.namespaces["pkg"] + assertNotNull(pkg) + assertTrue(pkg.isInferred) + + val pkgThirdModule = result.namespaces["pkg.third_module"] + assertNotNull(pkgThirdModule) + assertTrue(pkg.isInferred) + + val pkgFunction = result.functions["pkg.function"] + assertNotNull(pkgFunction) + assertTrue(pkg.isInferred) + + val anotherPkg = result.namespaces["another_pkg"] + assertNotNull(anotherPkg) + assertTrue(pkg.isInferred) + + val refs = result.refs + + // All reference except the .field access should be reference and not a member expression + refs.filter { it.name.localName != "field" }.forEach { assertIsNot(it) } + + assertEquals( + listOf( + // this is the default parameter of foo + "pkg.some_variable", + // lhs + "a", + // rhs, ME + "UNKNOWN.field", + // rhs, base of ME + "pkg.some_variable", + // lhs + "b", + // rhs + "pkg.function", + // lhs + "c", + // rhs + "another_pkg.function", + // lhs + "d", + // rhs + "another_pkg.function", + // lhs + "e", + // rhs + "pkg.third_module.variable", + ), + refs.map { it.name.toString() }, + ) } @Test diff --git a/cpg-language-python/src/test/resources/python/import_test.py b/cpg-language-python/src/test/resources/python/import_test.py index b8ba5b81486..9c0a195a174 100644 --- a/cpg-language-python/src/test/resources/python/import_test.py +++ b/cpg-language-python/src/test/resources/python/import_test.py @@ -1,5 +1,6 @@ import pkg.module from pkg import another_module +import pkg.another_module a = pkg.module.foo b = another_module.foo diff --git a/cpg-language-python/src/test/resources/python/import_vs_member.py b/cpg-language-python/src/test/resources/python/import_vs_member.py index 0b2e8e4cd10..d8d0d4f43e9 100644 --- a/cpg-language-python/src/test/resources/python/import_vs_member.py +++ b/cpg-language-python/src/test/resources/python/import_vs_member.py @@ -1,8 +1,27 @@ -from pkg import some_variable, function +from pkg import some_variable, function, third_module import another_pkg import another_pkg as alias +import pkg.third_module +# Here we should assume, that this is a member access of the field "field" on base "pkg.some_variable". +# We cannot see with certainty what "some_variable" is, but since we do not see any other import to it +# we can assume that it's not a module a = pkg.some_variable.field + +# Here more or less the same applies, we are importing a single symbol and use it as a function. We should +# infer the function "pkg.function". b = pkg.function() + +# Here we are importing the whole "another_pkg" module and using a function from it. We should infer the existance +# of "another_pkg" as namespace and "another_pkg.function" as a function c = another_pkg.function() -d = alias.function() \ No newline at end of file + +# This is just an alias of the above +d = alias.function() + +# This is a bit tricky, we should be able to see because of the last import in line 4 that "pkg.third_module" is a module +# and infer a namespace "pkg.third_module". This should then be a static reference to a variable in that module +e = third_module.variable + +def foo(bar = pkg.some_variable): + pass