New traversal proposal.

build.sbt

@@ -1,7 +1,7 @@
 name := "codepropertygraph"
 
 // parsed by project/Versions.scala, updated by updateDependencies.sh
-val overflowdbVersion = "1.64"
+val overflowdbVersion = "1.68"
 
 inThisBuild(
   List(
@@ -71,14 +71,14 @@ lazy val codepropertygraph = Projects.codepropertygraph
 lazy val semanticcpg = Projects.semanticcpg
 lazy val macros = Projects.macros
 lazy val schema2json = Projects.schema2json
+lazy val performance = Projects.performance
 
 // Once sbt-scalafmt is at version > 2.x, use scalafmtAll
 addCommandAlias("format", ";scalafixAll OrganizeImports;scalafmt;test:scalafmt")
 
 ThisBuild / scalacOptions ++= Seq(
   "-deprecation",
   "-feature",
-  "-Ywarn-unused", // required by scalafix
   "-Xfatal-warnings",
   "-language:implicitConversions",
   "-Ycache-macro-class-loader:last-modified",

performance/build.sbt

@@ -0,0 +1,6 @@
+name := "performance"
+
+dependsOn(Projects.semanticcpg)
+
+enablePlugins(JmhPlugin)
+

performance/src/main/scala/MyTests.scala

@@ -0,0 +1,170 @@
+package io.shiftleft.semanticcpg.language.types.expressions
+
+import io.shiftleft.codepropertygraph.generated.NodeTypes
+import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, Expression, Method, MethodParameterIn, MethodReturn, StoredNode}
+import io.shiftleft.semanticcpg.testing.MockCpg
+import io.shiftleft.semanticcpg.language.MySteps._
+import io.shiftleft.semanticcpg.language.ImportsV3._
+
+import scala.jdk.CollectionConverters._
+import org.openjdk.jmh.annotations._
+import MyTests._
+import io.shiftleft.semanticcpg.language.types.structure.LocalReferencingIdentifiers
+
+import scala.collection.{IterableOnceOps, SeqOps, View, mutable}
+import scala.collection.mutable.{ArrayBuffer, ListBuffer}
+import org.openjdk.jmh.infra.Blackhole
+import overflowdb.traversal.Traversal
+import some.SomeDomain._
+
+object MyTests {
+  import io.shiftleft.semanticcpg.language._
+  @State(Scope.Benchmark)
+  class MyState {
+    val (method, local) = {
+      val cpg = MockCpg()
+        .withMethod("methodForCfgTest")
+        .withLocalInMethod("methodForCfgTest", "x")
+        .withCallInMethod("methodForCfgTest", "call1")
+        .withCallInMethod("methodForCfgTest", "call2")
+        .withIdentifierArgument("call1", "x")
+        .cpg
+
+      (cpg.method.name("methodForCfgTest").head, cpg.local.name("x").head)
+    }
+    val d1 = D1(D2())
+  }
+}
+
+case class Bar[T](x: T)
+
+class MyTestNew {
+  import io.shiftleft.semanticcpg.langv2._
+  @Benchmark
+  def travNew(state: MyState) = {
+    val x = state.method.methodReturn
+    x
+  }
+
+  @Benchmark
+  def travBase(state: MyState) = {
+    val x = state.method._astOut.asScala.collectFirst { case x: MethodReturn => x }.get
+    x
+  }
+
+  @Benchmark
+  def refIdNew2(state: MyState) = {
+    val x = toLocalTraversalSingle(state.local).referencingIdentifiers
+    x
+  }
+
+  @Benchmark
+  def refIdNew3(state: MyState) = {
+    val x = Seq.from(rftoSingleExt(state.local).referencingIdentifiers)
+    x
+  }
+
+  @Benchmark
+  def refIdBase(state: MyState) = {
+    val x = Seq.from(state.local._refIn.asScala.filter(_.label == NodeTypes.IDENTIFIER))
+    x
+  }
+
+  @Benchmark
+  def allocTime1() = {
+    val x = mutable.ArrayBuffer.empty[Int]
+    x.append(1)
+    x
+  }
+
+  @Benchmark
+  def allocTime2() = {
+    Some(1)
+  }
+
+  @Benchmark
+  def allocTime3() = {
+    val x = ListBuffer.empty[Int]
+    x.append(1)
+    x
+  }
+
+  @Benchmark
+  def astTestNewV2(state: MyState) = {
+    val x: Option[Expression] = toAstNodeTraversalSingle(state.method).isExpression
+    x
+  }
+
+  @Benchmark
+  def astTestNewV3(state: MyState) = {
+    toSingleExt(state.method).isExpression
+  }
+
+  @Benchmark
+  def astAstNew(state: MyState) = {
+    state.method.ast
+  }
+
+  @Benchmark
+  def syntheticNew(state: MyState) = {
+    state.d1.toD2
+  }
+
+  @Benchmark
+  def syntheticBase(state: MyState) = {
+    state.d1.x
+  }
+
+  @Benchmark
+  def syntheticIterableNew(state: MyState) = {
+    //toSynth(state.d1:: Nil).toD2Multi
+    //val y: TravOps[Iterable, IterableTypes[Iterable, Iterable[Any]]] = toIt2Ops
+    val x: List[D2] = List(state.d1).toD2
+    x
+  }
+
+  @Benchmark
+  def syntheticIterableBase(state: MyState) = {
+    Iterable.single(state.d1).map(_.x)
+  }
+
+  def compileTest(state: MyState) = {
+    val a: D2 = state.d1.toD2
+    val b: Option[D2] = Option(state.d1).toD2
+    val c = toSynthIter(Array(state.d1).view.slice(1,2)).toD2
+    val d: View[D2] = c
+    val z: scala.collection.Seq[Int] = ArrayBuffer.empty[Int]
+
+    Iterable.single(state.d1).toD2Multi
+    Iterator.single(state.d1).toD2Multi
+
+    val a1 = Iterable.single(state.d1).doGlobal
+    val a2 = Iterator.single(state.d1).doGlobal
+  }
+}
+
+class MyTestsOld {
+  import io.shiftleft.semanticcpg.language._
+
+  @Benchmark
+  def travOld(state: MyState) = {
+    val x = toMethodMethods(state.method).methodReturn.l
+    x
+  }
+
+  @Benchmark
+  def refIdOld(state: MyState) = {
+    val x = Traversal(state.local).referencingIdentifiers2.toSeq
+    x
+  }
+
+  @Benchmark
+  def astTestOld(state: MyState) = {
+    Traversal(state.method).isExpression.headOption
+  }
+
+  @Benchmark
+  def astAstOld(state: MyState) = {
+    Traversal(state.method).ast.l
+  }
+}

performance/src/main/scala/some/SomeDomain.scala

@@ -0,0 +1,67 @@
+package some
+
+import io.shiftleft.semanticcpg.langv2._
+import some.SomeDomain.D1
+
+import scala.collection.IterableOnceOps
+
+
+object SomeDomain {
+    implicit def toSynthSingle[I <: D1](p: I): SynthExt[I, Single, Nothing] = {
+      new SynthExt(p: Single[I])
+    }
+
+    implicit def toSynthOption[I <: D1](trav: Option[I]): SynthExt[I, Option, Nothing] = {
+      new SynthExt(trav)
+    }
+
+    implicit def toSynthIter[I <: D1, CC[_], C](trav: IterableOnceOps[I, CC, C])
+    : SynthExt[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = {
+      new SynthExt[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav)
+    }
+
+  class SynthExt[I <: D1, IT[_], Marker](val in: IT[I]) extends AnyVal {
+    def toD2(implicit applier: ToOne[IT, Marker]) = {
+      applier.apply(in)(_.x)
+    }
+    def toD2Multi(implicit applier: ToMany[IT, Marker]) = {
+      applier.apply(in)(Iterator.single)
+    }
+    def doGlobal(implicit applier: ToGlobal[IT, Marker]) = {
+      applier.apply(in)(x => x)
+    }
+  }
+
+  case class D1(x: D2)
+
+  case class D2()
+}
+/*
+object SomeDomain2 {
+  trait SyncExtImplicits {
+    implicit def toSynthExtSingle(p: D1) = {
+      new SynthExt[Single, Single, Option, Option, Seq](p: Single[D1])
+    }
+
+    implicit def toSynthExtOption[T[_]](trav: Option[D1]) = {
+      new SynthExt[Option, Option, Option, Option, Seq](trav)
+    }
+
+    implicit def toSynthExtIter[CC[_], C](trav: IterableOnceOps[D1, CC, C])
+    : SynthExt[({type X[B] = IterableOnceOps[B, CC, C]})#X, CC, CC, ({type X[B] = C})#X, CC] = {
+      new SynthExt[({type X[B] = IterableOnceOps[B, CC, C]})#X, CC, CC, ({type X[B] = C})#X, CC](trav)
+    }
+  }
+
+  class SynthExt[Collection[_], CCOneToOne[_], CCOneToOption[_], CCOneToBoolean[_], CCOneToMany[_]]
+  (val trav: Collection[D1]) extends AnyVal {
+    def toD2(implicit ops: TravOpsToOne[Collection]) = {
+      trav.map(_.x)
+    }
+
+    def toD2Multi(implicit ops: TravOpsToOne[Collection]) = {
+      trav.flatMap(Iterator.single)
+    }
+  }
+}
+ */

project/Projects.scala

@@ -8,4 +8,5 @@ object Projects {
   lazy val semanticcpg = project.in(file("semanticcpg"))
   lazy val macros = project.in(file("macros"))
   lazy val schema2json = project.in(file("schema2json"))
+  lazy val performance = project.in(file("performance"))
 }

project/plugins.sbt

@@ -8,3 +8,4 @@ addSbtPlugin("com.dwijnand" % "sbt-dynver" % "3.1.0")
 addSbtPlugin("com.typesafe.sbt"  % "sbt-native-packager"   % "1.3.5")
 addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.9.27")
 addSbtPlugin("io.shiftleft" % "sbt-overflowdb" % "2.9")
+addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.3")

semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/ImportsV3.scala

@@ -0,0 +1,8 @@
+package io.shiftleft.semanticcpg.language
+
+import io.shiftleft.semanticcpg.language.types.expressions.generalizations.AstNodeIsExpression
+import io.shiftleft.semanticcpg.language.types.structure.LocalReferencingIdentifiers
+
+object ImportsV3 extends AstNodeIsExpression.Imports with LocalReferencingIdentifiers.Imports {
+
+}

semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/MySteps.scala

@@ -0,0 +1,79 @@
+package io.shiftleft.semanticcpg.language
+
+import io.shiftleft.semanticcpg.language.MySteps._
+import io.shiftleft.codepropertygraph.generated.nodes
+import io.shiftleft.semanticcpg.language.types.structure.LocalTraversalNew3
+import overflowdb.traversal.Traversal
+
+import scala.jdk.CollectionConverters._
+
+// First attempt of new pipeline. Not further used since it is not so fast.
+
+trait EPipe[I] {
+  type ColT[_]
+  protected val p: ColT[I]
+  protected val ops: PipeOps[ColT]
+
+  def head: I = {
+    ops.head(p)
+  }
+
+  def map[O](f: I => O): EPipe[O] = {
+    new Pipe(ops.map(p)(f), ops.T1to1FollowUpOps)
+  }
+
+  def filter2(f: I => Boolean): EPipe[I] = {
+    new Pipe(ops.filter(p)(f), ops.T1toOptionFollowUpOps)
+  }
+
+  def flatMap[O](f: I => Iterable[O]): EPipe[O] = {
+    new Pipe(ops.flatMap(p)(f), ops.T1toNFollowUpOps)
+  }
+
+  def flatMapIter[O](f: I => Iterator[O]): EPipe[O] = {
+    new Pipe(ops.flatMapIter(p)(f), ops.T1toNFollowUpOps)
+  }
+
+  def cast[A <: I]: EPipe[A] = {
+    this.asInstanceOf[EPipe[A]]
+  }
+}
+
+class Pipe[I, T[_]](val p: T[I], val ops: PipeOps[T]) extends EPipe[I] {
+  override type ColT[A] = T[A]
+}
+
+object MySteps {
+  type Pipe1[T] = T
+
+  implicit val ops1 = Pipe1Ops
+
+  implicit val opsN = PipeNIterableOps
+
+  implicit def toExtClass[I <: nodes.Method](pipe: Iterable[I]): ExtensionClass[I] = {
+    new ExtensionClass(new Pipe(pipe, PipeNIterableOps))
+  }
+
+  implicit def toExtClass[I <: nodes.Method](pipe: I): ExtensionClass[I] = {
+    new ExtensionClass(new Pipe(pipe: Pipe1[I], Pipe1Ops))
+  }
+
+  implicit def toExtClass[I <: nodes.Method](pipe: EPipe[I]): ExtensionClass[I] = {
+    new ExtensionClass(pipe)
+  }
+
+  implicit def toLocalNew[I <: nodes.Local](pipe: I): LocalTraversalNew3 = {
+    new LocalTraversalNew3(new Pipe(pipe: Pipe1[I], Pipe1Ops))
+  }
+
+}
+
+class ExtensionClass[I <: nodes.Method](val pipe: EPipe[I]) extends AnyVal {
+  def methodReturn2() = {
+    pipe.map(_._astOut.asScala.collectFirst { case x: nodes.MethodReturn => x }.get)
+  }
+
+  def methodParameters = {
+    pipe.flatMapIter(_._astOut.asScala.collect { case x: nodes.MethodParameterIn => x })
+  }
+}