Merging PythonAddDeclarationPass into SymbolResolver using HasDynamicDeclarations trait

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt

@@ -27,16 +27,20 @@ package de.fraunhofer.aisec.cpg.frontends
 
 import de.fraunhofer.aisec.cpg.ScopeManager
 import de.fraunhofer.aisec.cpg.TranslationContext
-import de.fraunhofer.aisec.cpg.graph.HasOperatorCode
-import de.fraunhofer.aisec.cpg.graph.HasOverloadedOperation
-import de.fraunhofer.aisec.cpg.graph.LanguageProvider
-import de.fraunhofer.aisec.cpg.graph.Name
+import de.fraunhofer.aisec.cpg.graph.*
+import de.fraunhofer.aisec.cpg.graph.declarations.Declaration
 import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration
 import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration
 import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration
-import de.fraunhofer.aisec.cpg.graph.scopes.*
+import de.fraunhofer.aisec.cpg.graph.edges.flows.*
+import de.fraunhofer.aisec.cpg.graph.scopes.GlobalScope
+import de.fraunhofer.aisec.cpg.graph.scopes.RecordScope
+import de.fraunhofer.aisec.cpg.graph.scopes.Scope
+import de.fraunhofer.aisec.cpg.graph.scopes.Symbol
 import de.fraunhofer.aisec.cpg.graph.statements.expressions.*
-import de.fraunhofer.aisec.cpg.passes.*
+import de.fraunhofer.aisec.cpg.passes.ResolveCallExpressionAmbiguityPass
+import de.fraunhofer.aisec.cpg.passes.ResolveMemberExpressionAmbiguityPass
+import de.fraunhofer.aisec.cpg.passes.SymbolResolver
 import kotlin.reflect.KClass
 
 /**
@@ -298,6 +302,26 @@ inline infix fun <reified T : HasOverloadedOperation> KClass<T>.of(
     return Pair(T::class, operatorCode)
 }
 
+/**
+ * A language trait that specifies that this language has dynamic declarations, meaning that
+ * declarations can be added to the symbol table at runtime. Since we are a static analysis tools,
+ * we can only deliver an approximation to the actual behaviour.
+ */
+interface HasDynamicDeclarations : LanguageTrait {
+
+    /**
+     * A callback that can be used by a language to provide a declaration for a reference. The
+     * language is responsible for
+     * - Adding the declaration to the symbol table (using [ScopeManager.addDeclaration]; as well as
+     *   adding it to the AST
+     * - Injecting the declaration into the EOG path (if necessary). Since this reference might be
+     *   part of different AST expressions different approaches might be necessary. The utility
+     *   functions [Node.insertNodeBeforeInEOGPath] and [Node.insertNodeAfterwardInEOGPath] should
+     *   be used.
+     */
+    fun SymbolResolver.provideDeclaration(ref: Reference): Declaration?
+}
+
 /** Checks whether the name for a function (as [CharSequence]) is a known operator name. */
 context(LanguageProvider)
 val CharSequence.isKnownOperatorName: Boolean

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt

@@ -91,3 +91,89 @@ class EvaluationOrders<NodeType : Node>(
         outgoing = outgoing,
     ),
     MirroredEdgeCollection<Node, EvaluationOrder>
+
+/**
+ * This function inserts the given [newNode] before the current node ([this]) in its existing EOG
+ * path.
+ *
+ * Before:
+ * ```
+ * <node1> -- EOG -->
+ *                    <node3>
+ * <node2> -- EOG -->
+ * ```
+ *
+ * We want to insert a new [EvaluationOrder] edge between all incoming edges of [this] (node3).
+ *
+ * Afterward:
+ * ```
+ * <node1> -- EOG -->
+ *                    <new node> -- EOG --> <node3>
+ * <node2> -- EOG -->
+ * ```
+ */
+fun Node.insertNodeBeforeInEOGPath(
+    newNode: Node,
+    builder: ((EvaluationOrder) -> Unit)? = null,
+): Boolean {
+    // Construct a new edge from the given node to the current node
+    val edge = EvaluationOrder(newNode, this).also(builder ?: {})
+
+    // Make a copy of the incoming edges of the current node and set the start of the new edge as
+    // the end
+    val copy = this.prevEOGEdges.toList()
+    copy.forEach { it.end = edge.start }
+
+    // Clear the incoming edges of the current node
+    this.prevEOGEdges.clear()
+
+    // Add the old edges as the previous edges of the new edge's start. We cannot use "addAll"
+    // because otherwise our mirroring will not be triggered.
+    copy.forEach { edge.start.prevEOGEdges += it }
+
+    // Add the new edge as a previous edge of the current node
+    return this.prevEOGEdges.add(edge)
+}
+
+/**
+ * This function inserts the given [newNode] after the current node ([this]) in its existing EOG
+ * path.
+ *
+ * Before:
+ * ```
+ *         -- EOG --> <node1>
+ * <node3>
+ *         -- EOG --> <node2>
+ * ```
+ *
+ * We want to insert a new [EvaluationOrder] edge between all outgoing edges of [this] (node3).
+ *
+ * Afterward:
+ * ```
+ *                               -- EOG --> <node1>
+ * <node3> -- EOG --> <new node>
+ *                               -- EOG --> <node2>
+ * ```
+ */
+fun Node.insertNodeAfterwardInEOGPath(
+    newNode: Node,
+    builder: ((EvaluationOrder) -> Unit)? = null,
+): Boolean {
+    // Construct a new edge from the current node to the given node
+    val edge = EvaluationOrder(this, newNode).also(builder ?: {})
+
+    // Make a copy of the outgoing edges of the current node and set the end of the new edge as
+    // the start
+    val copy = this.nextEOGEdges.toList()
+    copy.forEach { it.start = edge.end }
+
+    // Clear the outgoing edges of the current node
+    this.nextEOGEdges.clear()
+
+    // Add the old edges as the next edges of the new edge's end. We cannot use "addAll" because
+    // otherwise our mirroring will not be triggered.
+    copy.forEach { edge.end.nextEOGEdges += it }
+
+    // Add the new edge as a next edge of the current node
+    return this.nextEOGEdges.add(edge)
+}

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt

@@ -34,6 +34,7 @@ import de.fraunhofer.aisec.cpg.graph.Node
 import de.fraunhofer.aisec.cpg.graph.StatementHolder
 import de.fraunhofer.aisec.cpg.graph.declarations.*
 import de.fraunhofer.aisec.cpg.graph.edges.flows.EvaluationOrder
+import de.fraunhofer.aisec.cpg.graph.edges.flows.insertNodeBeforeInEOGPath
 import de.fraunhofer.aisec.cpg.graph.firstParentOrNull
 import de.fraunhofer.aisec.cpg.graph.statements.*
 import de.fraunhofer.aisec.cpg.graph.statements.expressions.*
@@ -1482,3 +1483,28 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa
         nextEdgeBranch = tmpBranchLabel
     }
 }
+
+/**
+ * This adds an extra declaration (see [AssignExpression.declarations] to the EOG of the
+ * [AssignExpression]. This is needed if we somehow add the declaration after the pass has initially
+ * run.
+ *
+ * This mimics the behaviour in [EvaluationOrderGraphPass.handleAssignExpression].
+ */
+fun AssignExpression.addDeclarationToEOG(decl: Declaration): Boolean? {
+    // We need to insert the declaration before the first lhs ref
+    return this.lhs.firstOrNull()?.insertNodeBeforeInEOGPath(decl)
+}
+
+/**
+ * This adds an extra local declaration (see [ForEachStatement.locals]) to the EOG of the
+ * [ForEachStatement]. This is needed if we somehow add the declaration after the pass has initially
+ * run.
+ *
+ * This mimics the behaviour in [EvaluationOrderGraphPass.handleForEachStatement].
+ */
+fun ForEachStatement.addDeclarationToEOG(decl: Declaration): Boolean? {
+    // Actually, it seems like we are not adding the declaration of a local variable to the EOG of
+    // the ForEachStatement at all.
+    return true
+}

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt

@@ -194,6 +194,13 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) {
         val language = ref.language
         val helperType = ref.resolutionHelper?.type
 
+        // Before we resolve anything, we give languages with dynamic declarations a chance to
+        // declare any variables they need. This is most likely only needed for WRITE references,
+        // but we let the language decide that
+        if (language is HasDynamicDeclarations) {
+            with(language) { provideDeclaration(ref) }
+        }
+
         // Ignore references to anonymous identifiers, if the language supports it (e.g., the _
         // identifier in Go)
         if (
@@ -312,6 +319,13 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) {
         val base = current.base
         val language = current.language
 
+        // Before we resolve anything, we give languages with dynamic declarations a chance to
+        // declare any variables they need. This is most likely only needed for WRITE references,
+        // but we let the language decide that
+        if (language is HasDynamicDeclarations) {
+            with(language) { provideDeclaration(current) }
+        }
+
         // We need to adjust certain types of the base in case of a "super" expression, and we
         // delegate this to the language. If that is successful, we can continue with regular
         // resolving.

cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrderTest.kt

@@ -34,18 +34,18 @@ class EvaluationOrderTest {
     fun testEvaluationOrders() {
         with(TestLanguageFrontend()) {
             // <node1> -- EOG --> <node2>
-            // this should be 1 nextDFG for node1 and 1 prevDFG for node2
-            var node1 = newLiteral(value = 1)
-            var node2 = newLiteral(value = 1)
-
+            // this should be 1 nextEOG for node1 and 1 prevEOG for node2
+            val node1 = newLiteral(value = 1)
+            val node2 = newLiteral(value = 2)
             node1.nextEOGEdges.add(node2)
-            // should contain 1 prevDFG edge now
+
+            // should contain 1 prevEOG edge now
             assertEquals(1, node2.prevEOGEdges.size)
-            // and it should be the same as the nextDFG of node1
+            // and it should be the same as the nextEOG of node1
             assertSame(node1.nextEOGEdges.firstOrNull(), node2.prevEOGEdges.firstOrNull())
 
             node1.nextEOGEdges.removeAt(0)
-            // should contain 0 prevDFG edge now
+            // should contain 0 prevEOG edge now
             assertEquals(0, node2.prevEOGEdges.size)
         }
     }
@@ -54,9 +54,9 @@ class EvaluationOrderTest {
     fun testClear() {
         with(TestLanguageFrontend()) {
             // <node1> -- EOG --> <node2>
-            // this should be 1 nextDFG for node1 and 1 prevDFG for node2
-            var node1 = newLiteral(value = 1)
-            var node2 = newLiteral(value = 1)
+            // this should be 1 nextEOG for node1 and 1 prevEOG for node2
+            val node1 = newLiteral(value = 1)
+            val node2 = newLiteral(value = 2)
 
             node1.nextEOGEdges.add(node2)
             assertEquals(1, node2.prevEOGEdges.size)
@@ -66,4 +66,103 @@ class EvaluationOrderTest {
             assertEquals(0, node2.prevEOGEdges.size)
         }
     }
+
+    @Test
+    fun testInsertBefore() {
+        with(TestLanguageFrontend()) {
+            val node1 = newLiteral(value = 1)
+            val node2 = newLiteral(value = 2)
+            val node3 = newLiteral(value = 3)
+
+            // <node1> -- EOG -->
+            //                    <node3>
+            // <node2> -- EOG -->
+            node1.nextEOGEdges += node3
+            node2.nextEOGEdges += node3
+            assertEquals(2, node3.prevEOGEdges.size, "node3 should contain 2 prevEOG edges now")
+            assertEquals(
+                setOf(node1, node2),
+                node3.prevEOG.toSet(),
+                "node3 should have node1 and node2 as prevEOG",
+            )
+
+            // Now let's insert an edge from node4 to node3 "before" node3
+            // <node1> -- EOG -->
+            //                    <node4> -- EOG --> <node3>
+            // <node2> -- EOG -->
+            val node4 = newLiteral(value = 4)
+            assertTrue(node3.insertNodeBeforeInEOGPath(node4))
+            assertEquals(1, node3.prevEOGEdges.size, "node3 should contain 1 prevEOG edge now")
+            assertSame(
+                node4,
+                node3.prevEOGEdges.singleOrNull()?.start,
+                "node4 should be the start of the single prevEOG edge of node3",
+            )
+            assertEquals(2, node4.prevEOGEdges.size, "node4 should contain 2 prevEOG edges now")
+            assertEquals(
+                setOf(node1, node2),
+                node4.prevEOG.toSet(),
+                "node4 should have node1 and node2 as prevEOG",
+            )
+            assertEquals(1, node1.nextEOGEdges.size, "node1 should contain 1 nextEOG edge now")
+            assertSame(
+                node4,
+                node1.nextEOGEdges.singleOrNull()?.end,
+                "node4 should be the end of the single nextEOG edge of node1",
+            )
+        }
+    }
+
+    @Test
+    fun testInsertAfterward() {
+        with(TestLanguageFrontend()) {
+            val node1 = newLiteral(value = 1)
+            val node2 = newLiteral(value = 2)
+            val node3 = newLiteral(value = 3)
+
+            //         -- EOG --> <node1>
+            // <node3>
+            //         -- EOG --> <node2>
+            //
+            node3.nextEOGEdges += node1
+            node3.nextEOGEdges += node2
+            assertEquals(2, node3.nextEOGEdges.size, "node3 should contain 2 nextEOG edges now")
+            assertEquals(
+                setOf(node1, node2),
+                node3.nextEOG.toSet(),
+                "node3 should have node1 and node2 as nextEOG",
+            )
+
+            // Now let's insert an edge from node3 to node4 "after" node3
+            //                               -- EOG --> <node1>
+            // <node3> -- EOG --> <node4>
+            //                               -- EOG --> <node2>
+            val node4 = newLiteral(value = 4)
+            assertTrue(node3.insertNodeAfterwardInEOGPath(node4))
+            assertEquals(1, node3.nextEOGEdges.size, "node3 should contain 1 nextEOG edge now")
+            assertSame(
+                node4,
+                node3.nextEOGEdges.singleOrNull()?.end,
+                "node4 should be the end of the single nextEOG edge of node3",
+            )
+            assertEquals(2, node4.nextEOGEdges.size, "node4 should contain 2 nextEOG edges now")
+            assertEquals(
+                setOf(node1, node2),
+                node4.nextEOG.toSet(),
+                "node4 should have node1 and node2 as nextEOG",
+            )
+            assertEquals(1, node1.prevEOGEdges.size, "node1 should contain 1 prevEOG edge now")
+            assertSame(
+                node4,
+                node1.prevEOGEdges.singleOrNull()?.start,
+                "node4 should be the start of the single prevEOG edge of node1",
+            )
+            assertEquals(1, node2.prevEOGEdges.size, "node2 should contain 1 prevEOG edge now")
+            assertSame(
+                node4,
+                node2.prevEOGEdges.singleOrNull()?.start,
+                "node4 should be the start of the single prevEOG edge of node2",
+            )
+        }
+    }
 }