[modulevariable] Replace Module Block Structure with Module Variable Steps

* Reverts #3750's module block structure in Python and performs more on-the-fly interprocedural usage resolution
* Adds `MODULE` modifier on module-defining methods in JS and Ruby (Python already has this). Note: Other frontends don't seem to have a module-defining method, though I may be wrong.
* Transferred import resolution query steps and classes to `semanticcpg` under `importresolver` to perform on-the-fly CPG entity retrieval of resolved imports.
* Modified existing `modulevariable` query steps to now reflect `LOCAL` as the base of the `ModuleVariable` node extension and created a bunch of queries around that.
* Modified `SourcesToStartingPoints` to handle module variables similarly to fields for interprocedural flows.

Author: DavidBakerEffendi

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/package.scala

@@ -5,9 +5,8 @@ import io.shiftleft.semanticcpg.language.*
 
 package object dataflowengineoss {
 
-  def globalFromLiteral(lit: Literal): Iterator[Expression] = lit.start
-    .where(_.inAssignment.method.nameExact("<module>", ":package"))
-    .inAssignment
+  def globalFromLiteral(lit: Literal): Iterator[Expression] = lit.start.inAssignment
+    .where(_.method.isModule)
     .argument(1)
 
   def identifierToFirstUsages(node: Identifier): List[Identifier] = node.refsTo.flatMap(identifiersFromCapturedScopes).l

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/SourcesToStartingPoints.scala

@@ -3,9 +3,10 @@ package io.joern.dataflowengineoss.queryengine
 import io.joern.dataflowengineoss.globalFromLiteral
 import io.joern.x2cpg.Defines
 import io.shiftleft.codepropertygraph.Cpg
-import io.shiftleft.codepropertygraph.generated.Operators
-import io.shiftleft.codepropertygraph.generated.nodes._
-import io.shiftleft.semanticcpg.language._
+import io.shiftleft.codepropertygraph.generated.nodes.*
+import io.shiftleft.semanticcpg.language.*
+import io.shiftleft.semanticcpg.language.importresolver.{EvaluatedImport, ResolvedMember, ResolvedTypeDecl}
+import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.Assignment
 import io.shiftleft.semanticcpg.language.operatorextension.allAssignmentTypes
 import io.shiftleft.semanticcpg.utils.MemberAccess.isFieldAccess
 import org.slf4j.LoggerFactory
@@ -63,6 +64,16 @@ class SourceTravsToStartingPointsTask[NodeType](sourceTravs: IterableOnce[NodeTy
 class SourceToStartingPoints(src: StoredNode) extends RecursiveTask[List[CfgNode]] {
 
   private val cpg = Cpg(src.graph())
+  private lazy val memberToImportingModule: Map[Member, List[(String, Method)]] = cpg.call
+    .where(_.method.isModule)
+    .flatMap(x =>
+      x.referencedImports
+        .flatMap(extractAliasMemberPair)
+        .map { case (alias, member) => member -> (alias, x.method) }
+    )
+    .groupBy(_._1)
+    .map { case (member, xs) => member -> xs.map(_._2) }
+  private val typeDeclToMembers = cpg.typeDecl.map { x => x.fullName -> x.member.l }.toMap
 
   override def compute(): List[CfgNode] = sourceToStartingPoints(src)
 
@@ -71,7 +82,12 @@ class SourceToStartingPoints(src: StoredNode) extends RecursiveTask[List[CfgNode
       case methodReturn: MethodReturn =>
         methodReturn.method.callIn.l
       case lit: Literal =>
-        List(lit) ++ usages(targetsToClassIdentifierPair(literalToInitializedMembers(lit))) ++ globalFromLiteral(lit)
+        val uses = usages(targetsToClassIdentifierPair(literalToInitializedMembers(lit)))
+        val globals = globalFromLiteral(lit).flatMap {
+          case x: Identifier if x.isModuleVariable => x +: moduleVarToUsages(x)
+          case x                                   => x :: Nil
+        }.l
+        lit :: (uses ++ globals)
       case member: Member =>
         usages(targetsToClassIdentifierPair(List(member)))
       case x: Declaration =>
@@ -91,17 +107,59 @@ class SourceToStartingPoints(src: StoredNode) extends RecursiveTask[List[CfgNode
 
   private def withFieldAndIndexAccesses(nodes: List[CfgNode]): List[CfgNode] =
     nodes.flatMap {
-      case identifier: Identifier =>
-        List(identifier) ++ fieldAndIndexAccesses(identifier)
-      case x => List(x)
+      case moduleVar: Identifier if moduleVar.isModuleVariable => moduleVar :: moduleVarToUsages(moduleVar)
+      case identifier: Identifier                              => identifier :: fieldAndIndexAccesses(identifier)
+      case x                                                   => x :: Nil
     }
 
   private def fieldAndIndexAccesses(identifier: Identifier): List[CfgNode] =
     identifier.method._identifierViaContainsOut
       .nameExact(identifier.name)
       .inCall
       .collect { case c if isFieldAccess(c.name) => c }
-      .l
+      .toList
+
+  private def extractAliasMemberPair(i: Import): Seq[(String, Member)] = {
+    i.importedAs
+      .map { alias =>
+        i.call.tag
+          .flatMap(EvaluatedImport.tagToEvaluatedImport)
+          .flatMap {
+            case ResolvedMember(basePath, memberName, _) =>
+              cpg.typeDecl.fullNameExact(basePath).member.nameExact(memberName).map(m => alias -> m)
+            case ResolvedTypeDecl(typeFullName, _) =>
+              cpg.typeDecl.fullNameExact(typeFullName).member.map(m => alias -> m)
+            case _ => Seq.empty
+          }
+          .toSeq
+      }
+      .getOrElse(Seq.empty)
+  }
+
+  private def moduleVarToUsages(moduleVar: Identifier): List[CfgNode] = {
+    typeDeclToMembers
+      .getOrElse(moduleVar.method.fullName, List.empty)
+      .nameExact(moduleVar.name)
+      .flatMap(memberToImportingModule.get)
+      .flatMap { xs =>
+        xs.flatMap { case (alias, importingModule) =>
+          val directAccess = importingModule
+            .flatMap(_._identifierViaContainsOut.nameExact(alias))
+            .sortBy(i => (i.lineNumber, i.columnNumber))
+            .filterNot(notLeftHandOfAssignment)
+            .headOption
+            .toList
+          val accessedAsFields = importingModule
+            .flatMap(_.fieldAccess.where(_.fieldIdentifier.canonicalNameExact(alias))) // TODO: This does not check LHS
+            .sortBy(i => (i.lineNumber, i.columnNumber))
+            .filterNot(notLeftHandOfAssignment)
+            .headOption
+            .toList
+          (directAccess ++ accessedAsFields).collectAll[CfgNode]
+        }
+      }
+      .toList
+  }
 
   private def usages(pairs: List[(TypeDecl, AstNode)]): List[CfgNode] = {
     pairs.flatMap { case (typeDecl, astNode) =>
@@ -184,8 +242,8 @@ class SourceToStartingPoints(src: StoredNode) extends RecursiveTask[List[CfgNode
               // If a member shares the name of the identifier then we consider this as a member
               lit.method.typeDecl.member.name.toSet.contains(identifier.name) =>
           List(identifier)
-        case call: Call if call.name == Operators.fieldAccess => call.ast.isFieldIdentifier.l
-        case _                                                => List[Expression]()
+        case call: Call if isFieldAccess(call.name) => call.ast.isFieldIdentifier.l
+        case _                                      => List[Expression]()
       }
       .l
 

joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreator.scala

@@ -7,9 +7,9 @@ import io.joern.jssrc2cpg.parser.BabelJsonParser.ParseResult
 import io.joern.jssrc2cpg.parser.BabelNodeInfo
 import io.joern.jssrc2cpg.passes.Defines
 import io.joern.x2cpg.datastructures.Stack.*
-import io.joern.x2cpg.utils.NodeBuilders.newMethodReturnNode
+import io.joern.x2cpg.utils.NodeBuilders.{newMethodReturnNode, newModifierNode}
 import io.joern.x2cpg.{Ast, AstCreatorBase, ValidationMode, AstNodeBuilder as X2CpgAstNodeBuilder}
-import io.shiftleft.codepropertygraph.generated.{EvaluationStrategies, NodeTypes}
+import io.shiftleft.codepropertygraph.generated.{EvaluationStrategies, ModifierTypes, NodeTypes}
 import io.shiftleft.codepropertygraph.generated.nodes.NewBlock
 import io.shiftleft.codepropertygraph.generated.nodes.NewFile
 import io.shiftleft.codepropertygraph.generated.nodes.NewMethod
@@ -124,7 +124,13 @@ class AstCreator(
     methodAstParentStack.pop()
 
     functionTypeAndTypeDeclAst.withChild(
-      methodAst(programMethod, List(Ast(thisParam)), blockAst(blockNode, methodChildren), methodReturn)
+      methodAst(
+        programMethod,
+        Ast(thisParam) :: Nil,
+        blockAst(blockNode, methodChildren),
+        methodReturn,
+        newModifierNode(ModifierTypes.MODULE) :: Nil
+      )
     )
   }
 

joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/passes/JavaScriptTypeRecovery.scala

@@ -33,6 +33,8 @@ private class JavaScriptTypeRecovery(cpg: Cpg, state: XTypeRecoveryState) extend
 private class RecoverForJavaScriptFile(cpg: Cpg, cu: File, builder: DiffGraphBuilder, state: XTypeRecoveryState)
     extends RecoverForXCompilationUnit[File](cpg, cu, builder, state) {
 
+  import io.joern.x2cpg.passes.frontend.XTypeRecovery.AllNodeTypesFromNodeExt
+
   override protected val pathSep = ':'
 
   /** A heuristic method to determine if a call is a constructor or not.

joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/dataflow/DataflowTest.scala

@@ -557,7 +557,8 @@ class DataflowTest extends DataFlowCodeToCpgSuite {
 
     val sink = cpg.call.nameExact("fn")
     val src  = cpg.literal("47")
-    sink.reachableBy(src).size shouldBe 1
+    // Deduplicated as flows skip over certain lowerings in other variants of the flows but source-sink pairs are equal
+    sink.reachableBy(src).dedup.size shouldBe 1
   }
 
   "Flow into method defined as lambda and assigned to constant" in {
@@ -679,13 +680,13 @@ class DataflowTest extends DataFlowCodeToCpgSuite {
     "literal to captured closure" in {
       val literalSource = cpg.literal.codeExact("\"https://test-api-service.com\"").l
       literalSource.size shouldBe 1
-      sink.reachableBy(literalSource).size shouldBe 1
+      sink.reachableBy(literalSource).dedup.size shouldBe 1
     }
 
     "identifiers to captured closure" in {
       val identifierSource = cpg.identifier.nameExact("API_Endpoint").lineNumber(5).l
       identifierSource.size shouldBe 1
-      sink.reachableBy(identifierSource).size shouldBe 1
+      sink.reachableBy(identifierSource).dedup.size shouldBe 1
     }
 
     "identifiers in the arg of the call" in {

joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/ImportResolverPass.scala

@@ -1,11 +1,11 @@
 package io.joern.pysrc2cpg
 
 import better.files.File
-import io.joern.x2cpg.passes.frontend.ImportsPass.*
 import io.joern.x2cpg.passes.frontend.XImportResolverPass
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes.*
 import io.shiftleft.semanticcpg.language.*
+import io.shiftleft.semanticcpg.language.importresolver.*
 
 import java.io.File as JFile
 import java.util.regex.Matcher

joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonImportResolverPass.scala

@@ -1,10 +1,11 @@
 package io.joern.pysrc2cpg
 
-import io.joern.x2cpg.passes.frontend._
+import io.joern.x2cpg.passes.frontend.*
 import io.shiftleft.codepropertygraph.Cpg
-import io.shiftleft.codepropertygraph.generated.nodes._
+import io.shiftleft.codepropertygraph.generated.nodes.*
 import io.shiftleft.codepropertygraph.generated.{Operators, PropertyNames}
-import io.shiftleft.semanticcpg.language._
+import io.shiftleft.semanticcpg.language.*
+import io.shiftleft.semanticcpg.language.importresolver.*
 import io.shiftleft.semanticcpg.language.operatorextension.OpNodes
 import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.FieldAccess
 import overflowdb.BatchedUpdate.DiffGraphBuilder
@@ -48,7 +49,6 @@ private class RecoverForPythonFile(cpg: Cpg, cu: File, builder: DiffGraphBuilder
 
   override def visitImport(i: Import): Unit = {
     if (i.importedAs.isDefined && i.importedEntity.isDefined) {
-      import io.joern.x2cpg.passes.frontend.ImportsPass.*
 
       val entityName = i.importedAs.get
       i.call.tag.flatMap(EvaluatedImport.tagToEvaluatedImport).foreach {
@@ -184,7 +184,8 @@ private class RecoverForPythonFile(cpg: Cpg, cu: File, builder: DiffGraphBuilder
     }
   }
 
-  override protected def postSetTypeInformation(): Unit =
+  override protected def postSetTypeInformation(): Unit = {
+    super.postSetTypeInformation()
     cu.typeDecl
       .map(t => t -> t.inheritsFromTypeFullName.partition(itf => symbolTable.contains(LocalVar(itf))))
       .foreach { case (t, (identifierTypes, otherTypes)) =>
@@ -195,6 +196,7 @@ private class RecoverForPythonFile(cpg: Cpg, cu: File, builder: DiffGraphBuilder
           builder.setNodeProperty(t, PropertyNames.INHERITS_FROM_TYPE_FULL_NAME, resolvedTypes)
         }
       }
+  }
 
   override def prepopulateSymbolTable(): Unit = {
     cu.ast.isMethodRef.where(_.astSiblings.isIdentifier.nameExact("classmethod")).referencedMethod.foreach {
@@ -218,4 +220,14 @@ private class RecoverForPythonFile(cpg: Cpg, cu: File, builder: DiffGraphBuilder
       .headOption
       .getOrElse(super.visitIdentifierAssignedToTypeRef(i, t, rec))
 
+  override protected def handlePotentialFunctionPointer(
+    funcPtr: Expression,
+    baseTypes: Set[String],
+    funcName: String,
+    baseName: Option[String]
+  ): Unit = {
+    if (funcName != "<module>")
+      super.handlePotentialFunctionPointer(funcPtr, baseTypes, funcName, baseName)
+  }
+
 }

joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonTypeRecovery.scala

@@ -5,8 +5,7 @@ import io.joern.dataflowengineoss.semanticsloader.FlowSemantic
 import io.joern.pysrc2cpg.PySrc2CpgFixture
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes.{Literal, Member, Method}
-import io.shiftleft.semanticcpg.language._
-import org.scalatest.Ignore
+import io.shiftleft.semanticcpg.language.*
 
 import java.io.File
 
@@ -589,4 +588,48 @@ class RegexDefinedFlowsDataFlowTests
     flows.size shouldBe 2
   }
 
+  "flow across interprocedural module variables" should {
+
+    "handle simple import and field-based usage of a literal" in {
+      val cpg: Cpg = code(
+        """
+          |a = 42
+          |""".stripMargin,
+        "foo.py"
+      )
+        .moreCode(
+          """
+            |import foo
+            |
+            |print(foo.a)
+            |""".stripMargin,
+          "bar.py"
+        )
+      val source = cpg.literal("42").l
+      val sink   = cpg.call.name("print.*").l
+      sink.reachableByFlows(source).size shouldBe 1
+    }
+
+    "handle simple import and aliased usage of a literal" in {
+      val cpg: Cpg = code(
+        """
+          |a = 42
+          |""".stripMargin,
+        "foo.py"
+      )
+        .moreCode(
+          """
+            |from foo import a as b
+            |
+            |print(b)
+            |""".stripMargin,
+          "bar.py"
+        )
+      val source = cpg.literal("42").l
+      val sink   = cpg.call.name("print.*").l
+      sink.reachableByFlows(source).size shouldBe 1
+    }
+
+  }
+
 }