From f4b6b086a62fb01d533e3f26fdb8070e5a9752c6 Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Thu, 11 Nov 2021 16:05:24 +0100 Subject: [PATCH 01/27] New traversal proposal. --- build.sbt | 4 +- performance/build.sbt | 6 + performance/src/main/scala/MyTests.scala | 153 ++++++++++++++++++ .../src/main/scala/some/SomeDomain.scala | 23 +++ project/Projects.scala | 1 + project/plugins.sbt | 1 + .../semanticcpg/language/MySteps.scala | 79 +++++++++ .../shiftleft/semanticcpg/language/New.scala | 31 ++++ .../semanticcpg/language/PipeOps.scala | 123 ++++++++++++++ .../generalizations/AstNodeTraversal.scala | 19 +++ .../types/structure/LocalTraversal.scala | 33 +++- .../types/structure/MethodTraversal.scala | 12 ++ .../semanticcpg/langv2/AnyTraversal.scala | 67 ++++++++ .../langv2/RepeatBehaviourBuilder.scala | 77 +++++++++ .../semanticcpg/langv2/TravOps.scala | 66 ++++++++ .../semanticcpg/langv2/package.scala | 14 ++ .../semanticcpg/testing/package.scala | 2 + 17 files changed, 708 insertions(+), 3 deletions(-) create mode 100644 performance/build.sbt create mode 100644 performance/src/main/scala/MyTests.scala create mode 100644 performance/src/main/scala/some/SomeDomain.scala create mode 100644 semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/MySteps.scala create mode 100644 semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala create mode 100644 semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/PipeOps.scala create mode 100644 semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala create mode 100644 semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/RepeatBehaviourBuilder.scala create mode 100644 semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala create mode 100644 semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala diff --git a/build.sbt b/build.sbt index 749bdde49..2528e6702 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ name := "codepropertygraph" // parsed by project/Versions.scala, updated by updateDependencies.sh -val overflowdbVersion = "1.64" +val overflowdbVersion = "1.68" inThisBuild( List( @@ -71,6 +71,7 @@ lazy val codepropertygraph = Projects.codepropertygraph lazy val semanticcpg = Projects.semanticcpg lazy val macros = Projects.macros lazy val schema2json = Projects.schema2json +lazy val performance = Projects.performance // Once sbt-scalafmt is at version > 2.x, use scalafmtAll addCommandAlias("format", ";scalafixAll OrganizeImports;scalafmt;test:scalafmt") @@ -78,7 +79,6 @@ addCommandAlias("format", ";scalafixAll OrganizeImports;scalafmt;test:scalafmt") ThisBuild / scalacOptions ++= Seq( "-deprecation", "-feature", - "-Ywarn-unused", // required by scalafix "-Xfatal-warnings", "-language:implicitConversions", "-Ycache-macro-class-loader:last-modified", diff --git a/performance/build.sbt b/performance/build.sbt new file mode 100644 index 000000000..c6ed3dcaa --- /dev/null +++ b/performance/build.sbt @@ -0,0 +1,6 @@ +name := "performance" + +dependsOn(Projects.semanticcpg) + +enablePlugins(JmhPlugin) + diff --git a/performance/src/main/scala/MyTests.scala b/performance/src/main/scala/MyTests.scala new file mode 100644 index 000000000..9f861e357 --- /dev/null +++ b/performance/src/main/scala/MyTests.scala @@ -0,0 +1,153 @@ +package io.shiftleft.semanticcpg.language.types.expressions + +import io.shiftleft.codepropertygraph.generated.NodeTypes +import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, Method, MethodReturn} +import io.shiftleft.semanticcpg.testing.MockCpg +import io.shiftleft.semanticcpg.language.MySteps._ + +import scala.jdk.CollectionConverters._ +import org.openjdk.jmh.annotations._ +import MyTests._ + +import scala.collection.mutable +import scala.collection.mutable.{ArrayBuffer, ListBuffer} +import org.openjdk.jmh.infra.Blackhole +import overflowdb.traversal.Traversal +import some.SomeDomain._ + +object MyTests { + import io.shiftleft.semanticcpg.language._ + @State(Scope.Benchmark) + class MyState { + val (method, local) = { + val cpg = MockCpg() + .withMethod("methodForCfgTest") + .withLocalInMethod("methodForCfgTest", "x") + .withCallInMethod("methodForCfgTest", "call1") + .withCallInMethod("methodForCfgTest", "call2") + .withIdentifierArgument("call1", "x") + .cpg + + (cpg.method.name("methodForCfgTest").head, cpg.local.name("x").head) + } + val d1 = D1(D2()) + } +} + +case class Bar[T](x: T) + +class MyTestNew { + import io.shiftleft.semanticcpg.langv2._ + import io.shiftleft.semanticcpg.language.New._ + @Benchmark + def newTrav(state: MyState) = { + val x = toExtClass(state.method).methodReturn2() + x + } + + @Benchmark + def newTrav2(state: MyState) = { + val x = state.method.methodReturn() + x + } + + @Benchmark + def direct2(state: MyState) = { + val x = state.method._astOut.asScala.collectFirst { case x: MethodReturn => x }.get + x + } + + @Benchmark + def refIdNew1(state: MyState) = { + val x = toLocalNew(state.local).referencingIdentifiers + x + } + + @Benchmark + def refIdNew2(state: MyState) = { + val x = toLocalTraversalNew1(state.local).referencingIdentifiers + x + } + + @Benchmark + def refIdBase(state: MyState) = { + val x = Iterable.from(state.local._refIn.asScala).filter(_.label == NodeTypes.IDENTIFIER) + x + } + + @Benchmark + def allocTime1() = { + val x = mutable.ArrayBuffer.empty[Int] + x.append(1) + x + } + + @Benchmark + def allocTime2() = { + Some(1) + } + + @Benchmark + def allocTime3() = { + val x = ListBuffer.empty[Int] + x.append(1) + x + } + + @Benchmark + def astTestNew(state: MyState) = { + state.method.isExpression.headOption + } + + @Benchmark + def astAstNew(state: MyState) = { + state.method.ast + } + + @Benchmark + def syntheticNew(state: MyState) = { + state.d1.toD2 + } + + @Benchmark + def syntheticBase(state: MyState) = { + state.d1.x + } + + @Benchmark + def syntheticIterableNew(state: MyState) = { + //toSynth(state.d1:: Nil).toD2 + Iterable.single(state.d1).toD2 + } + + @Benchmark + def syntheticIterableBase(state: MyState) = { + Iterable.single(state.d1).map(_.x) + } +} + +class MyTestsOld { + import io.shiftleft.semanticcpg.language._ + + @Benchmark + def oldTrav(state: MyState) = { + val x = toMethodMethods(state.method).methodReturn.l + x + } + + @Benchmark + def refIdOld(state: MyState) = { + val x = Traversal(state.local).referencingIdentifiers2.head + x + } + + @Benchmark + def astTestOld(state: MyState) = { + Traversal(state.method).isExpression.headOption + } + + @Benchmark + def astAstOld(state: MyState) = { + Traversal(state.method).ast.l + } +} diff --git a/performance/src/main/scala/some/SomeDomain.scala b/performance/src/main/scala/some/SomeDomain.scala new file mode 100644 index 000000000..9480a6e94 --- /dev/null +++ b/performance/src/main/scala/some/SomeDomain.scala @@ -0,0 +1,23 @@ +package some + +import io.shiftleft.semanticcpg.langv2._ + +object SomeDomain { + implicit def toSynth[FT[_]](p: D1): SynthExt[Trav1, FT] = { + new SynthExt(p: Trav1[D1]) + } + + implicit def toSynth[IT[_], FT[_]](p: IT[D1]): SynthExt[IT, FT] = { + new SynthExt(p) + } + + class SynthExt[IT[_], FT[_]](val trav: IT[D1]) extends AnyVal { + def toD2(implicit ops: TravOps[IT, FT]) = { + trav.map(_.x) + } + } + + case class D1(x: D2) + + case class D2() +} diff --git a/project/Projects.scala b/project/Projects.scala index 228f7db69..77ddc7c26 100644 --- a/project/Projects.scala +++ b/project/Projects.scala @@ -8,4 +8,5 @@ object Projects { lazy val semanticcpg = project.in(file("semanticcpg")) lazy val macros = project.in(file("macros")) lazy val schema2json = project.in(file("schema2json")) + lazy val performance = project.in(file("performance")) } diff --git a/project/plugins.sbt b/project/plugins.sbt index 0b7c329cc..7fa616d88 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -8,3 +8,4 @@ addSbtPlugin("com.dwijnand" % "sbt-dynver" % "3.1.0") addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.5") addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.9.27") addSbtPlugin("io.shiftleft" % "sbt-overflowdb" % "2.9") +addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.3") diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/MySteps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/MySteps.scala new file mode 100644 index 000000000..eef87f4dc --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/MySteps.scala @@ -0,0 +1,79 @@ +package io.shiftleft.semanticcpg.language + +import io.shiftleft.semanticcpg.language.MySteps._ +import io.shiftleft.codepropertygraph.generated.nodes +import io.shiftleft.semanticcpg.language.types.structure.LocalTraversalNew3 +import overflowdb.traversal.Traversal + +import scala.jdk.CollectionConverters._ + +// First attempt of new pipeline. Not further used since it is not so fast. + +trait EPipe[I] { + type ColT[_] + protected val p: ColT[I] + protected val ops: PipeOps[ColT] + + def head: I = { + ops.head(p) + } + + def map[O](f: I => O): EPipe[O] = { + new Pipe(ops.map(p)(f), ops.T1to1FollowUpOps) + } + + def filter2(f: I => Boolean): EPipe[I] = { + new Pipe(ops.filter(p)(f), ops.T1toOptionFollowUpOps) + } + + def flatMap[O](f: I => Iterable[O]): EPipe[O] = { + new Pipe(ops.flatMap(p)(f), ops.T1toNFollowUpOps) + } + + def flatMapIter[O](f: I => Iterator[O]): EPipe[O] = { + new Pipe(ops.flatMapIter(p)(f), ops.T1toNFollowUpOps) + } + + def cast[A <: I]: EPipe[A] = { + this.asInstanceOf[EPipe[A]] + } +} + +class Pipe[I, T[_]](val p: T[I], val ops: PipeOps[T]) extends EPipe[I] { + override type ColT[A] = T[A] +} + +object MySteps { + type Pipe1[T] = T + + implicit val ops1 = Pipe1Ops + + implicit val opsN = PipeNIterableOps + + implicit def toExtClass[I <: nodes.Method](pipe: Iterable[I]): ExtensionClass[I] = { + new ExtensionClass(new Pipe(pipe, PipeNIterableOps)) + } + + implicit def toExtClass[I <: nodes.Method](pipe: I): ExtensionClass[I] = { + new ExtensionClass(new Pipe(pipe: Pipe1[I], Pipe1Ops)) + } + + implicit def toExtClass[I <: nodes.Method](pipe: EPipe[I]): ExtensionClass[I] = { + new ExtensionClass(pipe) + } + + implicit def toLocalNew[I <: nodes.Local](pipe: I): LocalTraversalNew3 = { + new LocalTraversalNew3(new Pipe(pipe: Pipe1[I], Pipe1Ops)) + } + +} + +class ExtensionClass[I <: nodes.Method](val pipe: EPipe[I]) extends AnyVal { + def methodReturn2() = { + pipe.map(_._astOut.asScala.collectFirst { case x: nodes.MethodReturn => x }.get) + } + + def methodParameters = { + pipe.flatMapIter(_._astOut.asScala.collect { case x: nodes.MethodParameterIn => x }) + } +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala new file mode 100644 index 000000000..db705c8b8 --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala @@ -0,0 +1,31 @@ +package io.shiftleft.semanticcpg.language + +import io.shiftleft.codepropertygraph.generated.nodes +import io.shiftleft.semanticcpg.language.types.expressions.generalizations.AstTraversalNew +import io.shiftleft.semanticcpg.language.types.structure.{LocalTraversalNew, MethodTraversalNew} +import io.shiftleft.semanticcpg.langv2.Trav1 + +object New { + + implicit def toAstTraversalNew1[I <: nodes.AstNode, FT[_]](trav: I): AstTraversalNew[I, Trav1, FT] = { + new AstTraversalNew(trav: Trav1[I]) + } + implicit def toAstTraversalNew2[I <: nodes.AstNode, IT[_], FT[_]](trav: IT[I]): AstTraversalNew[I, IT, FT] = { + new AstTraversalNew(trav) + } + + implicit def toMethodTraversalNew1[I <: nodes.Method, FT[_]](trav: I): MethodTraversalNew[I, Trav1, FT] = { + new MethodTraversalNew(trav: Trav1[I]) + } + implicit def toMethodTraversalNew2[I <: nodes.Method, IT[_], FT[_]](trav: IT[I]): MethodTraversalNew[I, IT, FT] = { + new MethodTraversalNew(trav) + } + + implicit def toLocalTraversalNew1[I <: nodes.Local, FT[_]](trav: I): LocalTraversalNew[I, Trav1, FT] = { + new LocalTraversalNew(trav: Trav1[I]) + } + + implicit def toLocalTraversalNew2[I <: nodes.Local, IT[_], FT[_]](trav: IT[I]): LocalTraversalNew[I, IT, FT] = { + new LocalTraversalNew(trav) + } +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/PipeOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/PipeOps.scala new file mode 100644 index 000000000..7dca00684 --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/PipeOps.scala @@ -0,0 +1,123 @@ +package io.shiftleft.semanticcpg.language + +import io.shiftleft.semanticcpg.language.MySteps.Pipe1 + +// PipeOps for the new pipeline attempt in MySteps.scala. + +trait PipeOps[PipeT[_]] { + type T1to1[T] + type T1toOption[T] + type T1toN[T] + val T1to1FollowUpOps: PipeOps[T1to1] + val T1toOptionFollowUpOps: PipeOps[T1toOption] + val T1toNFollowUpOps: PipeOps[T1toN] + def head[I](pipe: PipeT[I]): I + def map[I, O](pipe: PipeT[I])(f: I => O): T1to1[O] + def filter[I](pipe: PipeT[I])(f: I => Boolean): T1toOption[I] + def flatMap[I, O](pipe: PipeT[I])(f: I => Iterable[O]): T1toN[O] + def flatMapIter[I, O](pipe: PipeT[I])(f: I => Iterator[O]): T1toN[O] +} + +object Pipe1Ops extends PipeOps[Pipe1] { + override type T1to1[T] = Pipe1[T] + override type T1toOption[T] = Option[T] + override type T1toN[T] = Iterable[T] + + override val T1to1FollowUpOps: PipeOps[T1to1] = Pipe1Ops + override val T1toOptionFollowUpOps: PipeOps[T1toOption] = PipeOptionOps + override val T1toNFollowUpOps: PipeOps[Iterable] = PipeNIterableOps + + override def head[I](pipe: Pipe1[I]): I = { + pipe + } + + override def map[I, O](pipe: Pipe1[I])(f: I => O): T1to1[O] = { + f(pipe) + } + + override def filter[I](pipe: Pipe1[I])(f: I => Boolean): T1toOption[I] = { + if (f(pipe)) { + Some(pipe) + } else { + None + } + } + + override def flatMap[I, O](pipe: Pipe1[I])(f: I => Iterable[O]): T1toN[O] = { + Iterable.from(f(pipe)) + } + + override def flatMapIter[I, O](pipe: Pipe1[I])(f: I => Iterator[O]): T1toN[O] = { + Iterable.from(f(pipe)) + } +} + +object PipeNIterableOps extends PipeOps[Iterable] { + override type T1to1[T] = Iterable[T] + override type T1toOption[T] = Iterable[T] + override type T1toN[T] = Iterable[T] + + override val T1to1FollowUpOps: PipeOps[T1to1] = PipeNIterableOps + override val T1toOptionFollowUpOps: PipeOps[T1toOption] = PipeNIterableOps + override val T1toNFollowUpOps: PipeOps[Iterable] = PipeNIterableOps + + override def head[I](pipe: Iterable[I]): I = { + pipe.head + } + + override def map[I, O](pipe: Iterable[I])(f: I => O): T1to1[O] = { + pipe.map(f) + } + + override def filter[I](pipe: Iterable[I])(f: I => Boolean): T1toOption[I] = { + pipe.filter(f) + } + + override def flatMap[I, O](pipe: Iterable[I])(f: I => Iterable[O]): T1toN[O] = { + pipe.flatMap(f) + } + + override def flatMapIter[I, O](pipe: Iterable[I])(f: I => Iterator[O]): T1toN[O] = { + pipe.flatMap(f) + } +} + +object PipeOptionOps extends PipeOps[Option] { + override type T1to1[T] = Option[T] + override type T1toOption[T] = Option[T] + override type T1toN[T] = Iterable[T] + + override val T1to1FollowUpOps: PipeOps[T1to1] = PipeOptionOps + override val T1toOptionFollowUpOps: PipeOps[T1toOption] = PipeOptionOps + override val T1toNFollowUpOps: PipeOps[Iterable] = PipeNIterableOps + + override def head[I](pipe: Option[I]): I = { + pipe.get + } + + override def map[I, O](pipe: Option[I])(f: I => O): T1to1[O] = { + pipe.map(f) + } + + override def filter[I](pipe: Option[I])(f: I => Boolean): T1toOption[I] = { + pipe.filter(f) + } + + override def flatMap[I, O](pipe: Option[I])(f: I => Iterable[O]): T1toN[O] = { + pipe match { + case Some(p) => + f(p) + case None => + Iterable.empty + } + } + + override def flatMapIter[I, O](pipe: Option[I])(f: I => Iterator[O]): T1toN[O] = { + pipe match { + case Some(p) => + Iterable.from(f(p)) + case None => + Iterable.empty + } + } +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala index a6c84d8eb..ed4e31257 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala @@ -3,8 +3,27 @@ package io.shiftleft.semanticcpg.language.types.expressions.generalizations import io.shiftleft.codepropertygraph.generated.nodes._ import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.langv2._ import overflowdb.traversal.help.Doc import overflowdb.traversal.{Traversal, help} +import scala.jdk.CollectionConverters._ + +class AstTraversalNew[I <: AstNode, IT[_], FT[_]](val trav: IT[I]) extends AnyVal { + + /** + * Nodes of the AST rooted in this node, including the node itself. + * */ + @Doc("All nodes of the abstract syntax tree") + def ast(implicit ops1: TravOps[IT, FT], ops2: TravNOps[FT]) = + trav.rFlatMap(_._astOut.asScala.asInstanceOf[Iterator[I]], _.emit).cast[AstNode] + + /** + * Traverse only to AST nodes that are expressions + * */ + def isExpression(implicit ops1: TravOps[IT, FT], ops2: TravNOps[FT]) = + trav.filter(_.isInstanceOf[Expression]).cast[Expression] + +} @help.Traversal(elementType = classOf[AstNode]) class AstNodeTraversal[A <: AstNode](val traversal: Traversal[A]) extends AnyVal { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala index 7beb8fa90..fb1b1550e 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala @@ -1,9 +1,24 @@ package io.shiftleft.semanticcpg.language.types.structure import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.nodes import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} +import io.shiftleft.semanticcpg.language.MySteps._ +import io.shiftleft.semanticcpg.language.EPipe +import io.shiftleft.semanticcpg.langv2._ import overflowdb.traversal.Traversal +import scala.jdk.CollectionConverters._ + +class LocalTraversalNew[I <: nodes.Local, IT[_], FT[_]](val pipe: IT[I]) extends AnyVal { + def referencingIdentifiers(implicit ops1: TravOps[IT, FT], ops2: TravNOps[FT]) = { + pipe + .flatMap(_._refIn.asScala) + .filter(_.label == NodeTypes.IDENTIFIER) + .cast[Identifier] + } +} + /** * A local variable * */ @@ -13,7 +28,7 @@ class LocalTraversal(val traversal: Traversal[Local]) extends AnyVal { * The method hosting this local variable * */ def method: Traversal[Method] = { - // TODO The following line of code is here for backwards compatibility. + // FTDO The following line of code is here for backwards compatibility. // Use the lower commented out line once not required anymore. traversal.repeat(_.in(EdgeTypes.AST))(_.until(_.hasLabel(NodeTypes.METHOD))).cast[Method] //definingBlock.method @@ -31,6 +46,9 @@ class LocalTraversal(val traversal: Traversal[Local]) extends AnyVal { def referencingIdentifiers: Traversal[Identifier] = traversal.in(EdgeTypes.REF).hasLabel(NodeTypes.IDENTIFIER).cast[Identifier] + def referencingIdentifiers2: Traversal[Identifier] = + traversal.flatMap(_._refIn.asScala).filter(_.label == NodeTypes.IDENTIFIER).cast[Identifier] + /** * The type of the local. * @@ -39,3 +57,16 @@ class LocalTraversal(val traversal: Traversal[Local]) extends AnyVal { def typ: Traversal[Type] = traversal.out(EdgeTypes.EVAL_TYPE).cast[Type] } + +class LocalTraversalNew3(val nodeOrPipe: EPipe[Local]) extends AnyVal { + def definingBlock = { + nodeOrPipe.map(_._astIn.asScala.next()).cast[Block] + } + + def referencingIdentifiers = { + nodeOrPipe + .flatMapIter(_._refIn.asScala) + .filter2(_.label == NodeTypes.IDENTIFIER) + .cast[Identifier] + } +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTraversal.scala index bea14f76d..ade03a7c3 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTraversal.scala @@ -3,9 +3,21 @@ package io.shiftleft.semanticcpg.language.types.structure import io.shiftleft.codepropertygraph.generated._ import io.shiftleft.codepropertygraph.generated.nodes._ import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.langv2._ import overflowdb._ import overflowdb.traversal.help.Doc import overflowdb.traversal.{Traversal, help} +import scala.jdk.CollectionConverters._ + +class MethodTraversalNew[I <: nodes.Method, IT[_], FT[_]](val trav: IT[I]) extends AnyVal { + def methodReturn()(implicit ops1: TravOps[IT, FT], ops2: TravNOps[FT]) = { + trav.map(_._astOut.asScala.collectFirst { case x: nodes.MethodReturn => x }.get) + } + + def methodParameters()(implicit ops: TravOps[IT, FT], ops2: TravNOps[FT]) = { + trav.flatMap(_._astOut.asScala.collect { case x: nodes.MethodParameterIn => x }) + } +} /** * A method, function, or procedure diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala new file mode 100644 index 000000000..24500fbd5 --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala @@ -0,0 +1,67 @@ +package io.shiftleft.semanticcpg.langv2 + +import overflowdb.traversal.RepeatStepIterator + +import scala.reflect.ClassTag + +class AnyTraversal[I, IT[_], FT[_]](val trav: IT[I]) extends AnyVal { + def head(implicit ops: TravOps[IT, FT]): I = { + ops.head(trav) + } + + def headOption(implicit ops: TravOps[IT, FT]): Option[I] = { + ops.headOption(trav) + } + + def map[O](f: I => O)(implicit ops: TravOps[IT, FT]): IT[O] = { + ops.map(trav)(f) + } + + def filter(f: I => Boolean)(implicit ops: TravOps[IT, FT]): FT[I] = { + ops.filter(trav)(f) + } + + def flatMap[O](f: I => Iterator[O])(implicit ops: TravOps[IT, FT]): FT[O] = { + ops.flatMap(trav)(f) + } + + def cast[A]: IT[A] = { + trav.asInstanceOf[IT[A]] + } + + // This is about ten times slower than .filter(_.isInstanceOf[C]).cast[C] + // but thanks to JVM generics we cannot use isInstanceOf with type parameters. + // So use this method for convenience on a REPL and otherwise use filter. + def collectAll[C: ClassTag](implicit ops: TravOps[IT, FT]): FT[C] = { + ops + .filter(trav) { + case _: C => true + case _ => false + } + .cast[C] + } + + def rMap[O <: I](f: I => O, g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( + implicit ops: TravOps[IT, FT]): FT[I] = { + val behaviour = g(new RepeatBehaviourBuilder()).build + ops.flatMap(trav)(i => new RepeatStepIterator(i, f.andThen(Iterator.single), behaviour)) + } + + def rFlatMap[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( + implicit ops: TravOps[IT, FT]): FT[I] = { + val behaviour = g(new RepeatBehaviourBuilder()).build + ops.flatMap(trav)(i => new RepeatStepIterator(i, f, behaviour)) + } + + @deprecated("Use rFlatMap instead") + def repeat[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( + implicit ops: TravOps[IT, FT]): FT[I] = { + rFlatMap(f, g) + } + + @deprecated("Calls to this function can be omitted") + def l: IT[I] = { + trav + } + +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/RepeatBehaviourBuilder.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/RepeatBehaviourBuilder.scala new file mode 100644 index 000000000..2bfd65b8c --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/RepeatBehaviourBuilder.scala @@ -0,0 +1,77 @@ +package io.shiftleft.semanticcpg.langv2 + +import overflowdb.traversal.RepeatBehaviour +import overflowdb.traversal.RepeatBehaviour.SearchAlgorithm + +class RepeatBehaviourBuilder[T] { + private[this] var _shouldEmit: (T, Int) => Boolean = (_, _) => false + private[this] var _untilCondition: Option[T => Iterator[_]] = None + private[this] var _whileCondition: Option[T => Iterator[_]] = None + private[this] var _times: Option[Int] = None + private[this] var _dedupEnabled: Boolean = false + private[this] var _searchAlgorithm: SearchAlgorithm.Value = SearchAlgorithm.DepthFirst + + /* configure search algorithm to go "breadth first", rather than the default "depth first" */ + def breadthFirstSearch: RepeatBehaviourBuilder[T] = { + _searchAlgorithm = SearchAlgorithm.BreadthFirst + this + } + + def bfs: RepeatBehaviourBuilder[T] = breadthFirstSearch + + /* configure `repeat` step to emit everything along the way */ + def emit: RepeatBehaviourBuilder[T] = { + _shouldEmit = (_, _) => true + this + } + + /* configure `repeat` step to emit everything along the way, apart from the _first_ element */ + def emitAllButFirst: RepeatBehaviourBuilder[T] = { + _shouldEmit = (_, depth) => depth > 0 + this + } + + /* configure `repeat` step to emit whatever meets the given condition */ + def emit(condition: T => Iterator[_]): RepeatBehaviourBuilder[T] = { + _shouldEmit = (element, _) => condition(element).hasNext + this + } + + /* Configure `repeat` step to stop traversing when given condition-traversal has at least one result. + * The condition-traversal is only evaluated _after_ the first iteration, for classic repeat/until behaviour */ + def until(condition: T => Iterator[_]): RepeatBehaviourBuilder[T] = { + _untilCondition = Some(condition) + this + } + + /* Configure `repeat` step to stop traversing when given condition-traversal has at least one result. + * The condition-traversal is already evaluated at the first iteration, for classic while/repeat behaviour. + * + * n.b. the only reason not to call this `while` is to avoid using scala keywords, which would need to be quoted. */ + def whilst(condition: T => Iterator[_]): RepeatBehaviourBuilder[T] = { + _whileCondition = Some(condition) + this + } + + /* configure `repeat` step to perform the given amount of iterations */ + def times(value: Int): RepeatBehaviourBuilder[T] = { + _times = Some(value) + this + } + + def dedup: RepeatBehaviourBuilder[T] = { + _dedupEnabled = true + this + } + + def build: RepeatBehaviour[T] = { + new RepeatBehaviour[T] { + override val searchAlgorithm: SearchAlgorithm.Value = _searchAlgorithm + override val untilCondition = _untilCondition + override val whileCondition = _whileCondition + final override val times: Option[Int] = _times + final override val dedupEnabled = _dedupEnabled + override def shouldEmit(element: T, currentDepth: Int): Boolean = _shouldEmit(element, currentDepth) + } + } +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala new file mode 100644 index 000000000..b939898e0 --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala @@ -0,0 +1,66 @@ +package io.shiftleft.semanticcpg.langv2 + +/** Traversal operations trait. + * Implementations of this trait handle the basic traversal operations for the + * different kind of input traversal types. + * @tparam InputTraversal Input traversal type. Abbreviated in other generics as IT. + * @tparam FlatMapTraversal Flat map operation output traversal type. + * Abbreviated in other generics as FT. + */ +trait TravOps[InputTraversal[_], FlatMapTraversal[_]] { + def head[I](trav: InputTraversal[I]): I + def headOption[I](trav: InputTraversal[I]): Option[I] + def map[I, O](trav: InputTraversal[I])(f: I => O): InputTraversal[O] + def filter[I](trav: InputTraversal[I])(f: I => Boolean): FlatMapTraversal[I] + def flatMap[I, O](trav: InputTraversal[I])(f: I => Iterator[O]): FlatMapTraversal[O] +} + +object Trav1Ops extends TravOps[Trav1, Iterable] { + + override def head[I](trav: Trav1[I]): I = { + trav + } + + override def headOption[I](trav: Trav1[I]): Option[I] = { + Some(trav) + } + + override def map[I, O](trav: Trav1[I])(f: I => O): Trav1[O] = { + f(trav) + } + + override def filter[I](trav: Trav1[I])(f: I => Boolean): Iterable[I] = { + if (f(trav)) { + trav :: Nil + } else { + Nil + } + } + + override def flatMap[I, O](trav: Trav1[I])(f: I => Iterator[O]): Iterable[O] = { + Iterable.from(f(trav)) + } +} + +object IterableOps extends TravOps[Iterable, Iterable] { + + override def head[I](trav: Iterable[I]): I = { + trav.head + } + + override def headOption[I](trav: Iterable[I]): Option[I] = { + trav.headOption + } + + override def map[I, O](trav: Iterable[I])(f: I => O): Iterable[O] = { + trav.map(f) + } + + override def filter[I](trav: Iterable[I])(f: I => Boolean): Iterable[I] = { + trav.filter(f) + } + + override def flatMap[I, O](trav: Iterable[I])(f: I => Iterator[O]): Iterable[O] = { + trav.flatMap(f) + } +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala new file mode 100644 index 000000000..27ef03dda --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala @@ -0,0 +1,14 @@ +package io.shiftleft.semanticcpg + +package object langv2 { + type Trav1[T] = T + type TravNOps[T[_]] = TravOps[T, T] + + implicit val trav1Ops = Trav1Ops + implicit def iterableOps[IT[_] <: Iterable[_]] = IterableOps.asInstanceOf[TravOps[IT, Iterable]] + + implicit def toAnyTraversal[I, IT[_], FT[_]](p: IT[I]): AnyTraversal[I, IT, FT] = { + new AnyTraversal(p) + } + +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/package.scala index d053c0aee..d9d9e106d 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/package.scala @@ -178,12 +178,14 @@ package object testing { val callNode = cpg.call.name(callName).head val methodNode = callNode.method.head val identifierNode = NewIdentifier().name(name) + val localNode = cpg.local.name(name).head val typeDecl = NewTypeDecl().name("abc") graph.addNode(identifierNode) graph.addNode(typeDecl) graph.addEdge(callNode, identifierNode, EdgeTypes.AST) graph.addEdge(methodNode, identifierNode, EdgeTypes.CONTAINS) graph.addEdge(identifierNode, typeDecl, EdgeTypes.REF) + graph.addEdge(identifierNode, localNode, EdgeTypes.REF) } def withCustom(f: (DiffGraph.Builder, Cpg) => Unit): MockCpg = { From abd065b88fec33feee5ae1f4719c015e228699ce Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Fri, 12 Nov 2021 10:56:28 +0100 Subject: [PATCH 02/27] Renaming of variable names to match schema. --- .../semanticcpg/language/types/structure/LocalTraversal.scala | 4 ++-- .../language/types/structure/MethodTraversal.scala | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala index fb1b1550e..adf9247dd 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala @@ -10,9 +10,9 @@ import overflowdb.traversal.Traversal import scala.jdk.CollectionConverters._ -class LocalTraversalNew[I <: nodes.Local, IT[_], FT[_]](val pipe: IT[I]) extends AnyVal { +class LocalTraversalNew[I <: nodes.Local, IT[_], FT[_]](val trav: IT[I]) extends AnyVal { def referencingIdentifiers(implicit ops1: TravOps[IT, FT], ops2: TravNOps[FT]) = { - pipe + trav .flatMap(_._refIn.asScala) .filter(_.label == NodeTypes.IDENTIFIER) .cast[Identifier] diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTraversal.scala index ade03a7c3..12b5e4cea 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTraversal.scala @@ -14,7 +14,7 @@ class MethodTraversalNew[I <: nodes.Method, IT[_], FT[_]](val trav: IT[I]) exten trav.map(_._astOut.asScala.collectFirst { case x: nodes.MethodReturn => x }.get) } - def methodParameters()(implicit ops: TravOps[IT, FT], ops2: TravNOps[FT]) = { + def methodParameters()(implicit ops1: TravOps[IT, FT], ops2: TravNOps[FT]) = { trav.flatMap(_._astOut.asScala.collect { case x: nodes.MethodParameterIn => x }) } } From 3a8f432bba0fda92ef5bb44f4e225b0dd1be05de Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Tue, 16 Nov 2021 13:00:59 +0100 Subject: [PATCH 03/27] Tmp --- performance/src/main/scala/MyTests.scala | 19 ++++++- .../semanticcpg/language/ImportsV3.scala | 8 +++ .../generalizations/AstNodeTraversal.scala | 41 +++++++++++++ .../types/structure/LocalTraversal.scala | 36 ++++++++++++ .../semanticcpg/langv2/package.scala | 5 ++ .../shiftleft/semanticcpg/langv3/Kernel.scala | 57 +++++++++++++++++++ 6 files changed, 163 insertions(+), 3 deletions(-) create mode 100644 semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/ImportsV3.scala create mode 100644 semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv3/Kernel.scala diff --git a/performance/src/main/scala/MyTests.scala b/performance/src/main/scala/MyTests.scala index 9f861e357..545cc4bb7 100644 --- a/performance/src/main/scala/MyTests.scala +++ b/performance/src/main/scala/MyTests.scala @@ -1,13 +1,15 @@ package io.shiftleft.semanticcpg.language.types.expressions import io.shiftleft.codepropertygraph.generated.NodeTypes -import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, Method, MethodReturn} +import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, Method, MethodReturn, StoredNode} import io.shiftleft.semanticcpg.testing.MockCpg import io.shiftleft.semanticcpg.language.MySteps._ +import io.shiftleft.semanticcpg.language.ImportsV3._ import scala.jdk.CollectionConverters._ import org.openjdk.jmh.annotations._ import MyTests._ +import io.shiftleft.semanticcpg.language.types.structure.LocalReferencingIdentifiers import scala.collection.mutable import scala.collection.mutable.{ArrayBuffer, ListBuffer} @@ -69,6 +71,12 @@ class MyTestNew { x } + @Benchmark + def refIdNew3(state: MyState) = { + val x = rftoSingleExt(state.local).referencingIdentifiers + x + } + @Benchmark def refIdBase(state: MyState) = { val x = Iterable.from(state.local._refIn.asScala).filter(_.label == NodeTypes.IDENTIFIER) @@ -95,8 +103,13 @@ class MyTestNew { } @Benchmark - def astTestNew(state: MyState) = { - state.method.isExpression.headOption + def astTestNewV2(state: MyState) = { + toAstTraversalNew1(state.method).isExpression + } + + @Benchmark + def astTestNewV3(state: MyState) = { + toSingleExt(state.method).isExpression } @Benchmark diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/ImportsV3.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/ImportsV3.scala new file mode 100644 index 000000000..02d62e94e --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/ImportsV3.scala @@ -0,0 +1,8 @@ +package io.shiftleft.semanticcpg.language + +import io.shiftleft.semanticcpg.language.types.expressions.generalizations.AstNodeIsExpression +import io.shiftleft.semanticcpg.language.types.structure.LocalReferencingIdentifiers + +object ImportsV3 extends AstNodeIsExpression.Imports with LocalReferencingIdentifiers.Imports { + +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala index ed4e31257..588cefec6 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala @@ -4,10 +4,51 @@ import io.shiftleft.codepropertygraph.generated.nodes._ import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} import io.shiftleft.semanticcpg.language._ import io.shiftleft.semanticcpg.langv2._ +import io.shiftleft.semanticcpg.langv3.{Helper, Kernel1ToN, Kernel1ToO} import overflowdb.traversal.help.Doc import overflowdb.traversal.{Traversal, help} + +import scala.collection.IterableOps import scala.jdk.CollectionConverters._ +class AstNodeIsExpressionKernel[I <: AstNode] extends Kernel1ToO[I, I with Expression] { + override def apply(i: I): Option[I with Expression] = { + if (i.isInstanceOf[Expression]) { + Some(i.asInstanceOf[I with Expression]) + } else { + None + } + } +} + +object AstNodeIsExpression { + type NodeType = AstNode + type KernelType[T <: NodeType] = AstNodeIsExpressionKernel[T] + + private val _impl = new KernelType[NodeType]() + private def impl[I <: NodeType] = _impl.asInstanceOf[KernelType[I]] + + trait Imports { + implicit def toSingleExt[I <: NodeType](i: I) = new SingleExt(i) + implicit def toIterableExt[I <: NodeType, CC[_], C](i: IterableOps[I, CC, C]) = new IterableExt(i) + implicit def toIteratorExt[I <: NodeType](i: Iterator[I]) = new IteratorExt(i) + implicit def toOptionExt[I <: NodeType](i: Option[I]) = new OptionExt(i) + } + + class SingleExt[I <: NodeType](val i: I) extends AnyVal { + def isExpression = Helper(i, impl[I]) + } + class IterableExt[I <: NodeType, CC[_], C](val i: IterableOps[I, CC, C]) extends AnyVal { + def isExpression = Helper(i, impl[I]) + } + class IteratorExt[I <: NodeType](val i: Iterator[I]) extends AnyVal { + def isExpression = Helper(i, impl[I]) + } + class OptionExt[I <: NodeType](val i: Option[I]) extends AnyVal { + def isExpression = Helper(i, impl[I]) + } +} + class AstTraversalNew[I <: AstNode, IT[_], FT[_]](val trav: IT[I]) extends AnyVal { /** diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala index adf9247dd..7e4b35cad 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala @@ -6,10 +6,46 @@ import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} import io.shiftleft.semanticcpg.language.MySteps._ import io.shiftleft.semanticcpg.language.EPipe import io.shiftleft.semanticcpg.langv2._ +import io.shiftleft.semanticcpg.langv3.{Helper, Kernel1ToN, Kernel1ToO} import overflowdb.traversal.Traversal +import scala.collection.IterableOps import scala.jdk.CollectionConverters._ +class LocalReferencingIdentifiersKernel[I <: nodes.Local] extends Kernel1ToN[I, nodes.Identifier] { + override def apply(i: I): Iterator[nodes.Identifier] = { + i._refIn.asScala.filter(_.label == NodeTypes.IDENTIFIER).asInstanceOf[Iterator[nodes.Identifier]] + } +} + +object LocalReferencingIdentifiers { + type NodeType = nodes.Local + type KernelType[T <: NodeType] = LocalReferencingIdentifiersKernel[T] + + private val _impl = new KernelType[NodeType]() + private def impl[I <: NodeType] = _impl.asInstanceOf[KernelType[I]] + + trait Imports { + implicit def rftoSingleExt[I <: NodeType](i: I) = new SingleExt(i) + implicit def rftoIterableExt[I <: NodeType, CC[_], C](i: IterableOps[I, CC, C]) = new IterableExt(i) + implicit def rftoIteratorExt[I <: NodeType](i: Iterator[I]) = new IteratorExt(i) + implicit def rftoOptionExt[I <: NodeType](i: Option[I]) = new OptionExt(i) + } + + class SingleExt[I <: NodeType](val i: I) extends AnyVal { + def referencingIdentifiers = Helper(i, impl[I]) + } + class IterableExt[I <: NodeType, CC[_], C](val i: IterableOps[I, CC, C]) extends AnyVal { + def referencingIdentifiers = Helper(i, impl[I]) + } + class IteratorExt[I <: NodeType](val i: Iterator[I]) extends AnyVal { + def referencingIdentifiers = Helper(i, impl[I]) + } + class OptionExt[I <: NodeType](val i: Option[I]) extends AnyVal { + def referencingIdentifiers = Helper(i, impl[I]) + } +} + class LocalTraversalNew[I <: nodes.Local, IT[_], FT[_]](val trav: IT[I]) extends AnyVal { def referencingIdentifiers(implicit ops1: TravOps[IT, FT], ops2: TravNOps[FT]) = { trav diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala index 27ef03dda..7a452e4f1 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala @@ -12,3 +12,8 @@ package object langv2 { } } +object foo { + implicit class MethoddParameterSingleExt(val node: Int) extends AnyVal { + def parameterB: Iterator[Int] = Iterator.single(2) + } +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv3/Kernel.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv3/Kernel.scala new file mode 100644 index 000000000..d5827f5e9 --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv3/Kernel.scala @@ -0,0 +1,57 @@ +package io.shiftleft.semanticcpg.langv3 + +import scala.collection.{IterableOps} + +trait Kernel1To1[-I, +O] extends Function1[I, O] +trait Kernel1ToN[-I, +O] extends Function1[I, Iterator[O]] +trait Kernel1ToO[-I, +O] extends Function1[I, Option[O]] + +object Helper { + def apply[I, O](x: I, kernel: Kernel1To1[I, O]): O = { + kernel.apply(x) + } + + def apply[I, O](x: I, kernel: Kernel1ToN[I, O]): Iterator[O] = { + kernel.apply(x) + } + + def apply[I, O](x: I, kernel: Kernel1ToO[I, O]): Option[O] = { + kernel.apply(x) + } + + def apply[I, CC[_], C, O](x: IterableOps[I, CC, C], kernel: Kernel1To1[I, O]): CC[O] = { + x.map(kernel) + } + + def apply[I, CC[_], C, O](x: IterableOps[I, CC, C], kernel: Kernel1ToN[I, O]): CC[O] = { + x.flatMap(kernel) + } + + def apply[I, CC[_], C, O](x: IterableOps[I, CC, C], kernel: Kernel1ToO[I, O]): CC[O] = { + x.flatMap(kernel) + } + + def apply[I, O](x: Iterator[I], kernel: Kernel1To1[I, O]): Iterator[O] = { + x.map(kernel) + } + + def apply[I, O](x: Iterator[I], kernel: Kernel1ToN[I, O]): Iterator[O] = { + x.flatMap(kernel) + } + + def apply[I, O](x: Iterator[I], kernel: Kernel1ToO[I, O]): Iterator[O] = { + x.flatMap(kernel) + } + + def apply[I, O](x: Option[I], kernel: Kernel1To1[I, O]): Option[O] = { + x.map(kernel) + } + + def apply[I, O](x: Option[I], kernel: Kernel1ToN[I, O]): Iterable[O] = { + x.map(y => Iterable.from(kernel(y))).getOrElse(Iterable.empty) + } + + def apply[I, O](x: Option[I], kernel: Kernel1ToO[I, O]): Option[O] = { + x.flatMap(kernel) + } +} From af9633291243ef1844fb07d538c854d688566387 Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Tue, 16 Nov 2021 17:05:26 +0100 Subject: [PATCH 04/27] Small stuff. --- performance/src/main/scala/MyTests.scala | 2 +- performance/src/main/scala/some/SomeDomain.scala | 3 +++ .../main/scala/io/shiftleft/semanticcpg/langv2/package.scala | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/performance/src/main/scala/MyTests.scala b/performance/src/main/scala/MyTests.scala index 545cc4bb7..29f55251b 100644 --- a/performance/src/main/scala/MyTests.scala +++ b/performance/src/main/scala/MyTests.scala @@ -129,7 +129,7 @@ class MyTestNew { @Benchmark def syntheticIterableNew(state: MyState) = { - //toSynth(state.d1:: Nil).toD2 + //toSynth(state.d1:: Nil).toD2Multi Iterable.single(state.d1).toD2 } diff --git a/performance/src/main/scala/some/SomeDomain.scala b/performance/src/main/scala/some/SomeDomain.scala index 9480a6e94..0377c0367 100644 --- a/performance/src/main/scala/some/SomeDomain.scala +++ b/performance/src/main/scala/some/SomeDomain.scala @@ -15,6 +15,9 @@ object SomeDomain { def toD2(implicit ops: TravOps[IT, FT]) = { trav.map(_.x) } + def toD2Multi(implicit ops: TravOps[IT, FT]) = { + trav.flatMap(x => Iterator.single(x)) + } } case class D1(x: D2) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala index 7a452e4f1..1d9dc98f0 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala @@ -5,7 +5,7 @@ package object langv2 { type TravNOps[T[_]] = TravOps[T, T] implicit val trav1Ops = Trav1Ops - implicit def iterableOps[IT[_] <: Iterable[_]] = IterableOps.asInstanceOf[TravOps[IT, Iterable]] + implicit def iterableOps[IT[_] <: Iterable[_]] = IterableOps.asInstanceOf[TravOps[IT, IT]] implicit def toAnyTraversal[I, IT[_], FT[_]](p: IT[I]): AnyTraversal[I, IT, FT] = { new AnyTraversal(p) From e0433a2466e642230ddb15240c7b14fdf67a22bd Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Wed, 17 Nov 2021 11:24:12 +0100 Subject: [PATCH 05/27] No more TravNOps and moved new Method traversal to separate file. --- performance/src/main/scala/MyTests.scala | 2 +- .../shiftleft/semanticcpg/language/New.scala | 11 +++--- .../generalizations/AstNodeTraversal.scala | 4 +-- .../types/structure/LocalTraversal.scala | 2 +- .../types/structure/MethodTraversal.scala | 12 ------- .../semanticcpg/langv2/package.scala | 10 ++---- .../types/structure/MethodTraversal.scala | 34 +++++++++++++++++++ 7 files changed, 46 insertions(+), 29 deletions(-) create mode 100644 semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala diff --git a/performance/src/main/scala/MyTests.scala b/performance/src/main/scala/MyTests.scala index 29f55251b..9190aded6 100644 --- a/performance/src/main/scala/MyTests.scala +++ b/performance/src/main/scala/MyTests.scala @@ -49,7 +49,7 @@ class MyTestNew { @Benchmark def newTrav2(state: MyState) = { - val x = state.method.methodReturn() + val x = state.method.methodReturn x } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala index db705c8b8..36c67255e 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala @@ -2,8 +2,9 @@ package io.shiftleft.semanticcpg.language import io.shiftleft.codepropertygraph.generated.nodes import io.shiftleft.semanticcpg.language.types.expressions.generalizations.AstTraversalNew -import io.shiftleft.semanticcpg.language.types.structure.{LocalTraversalNew, MethodTraversalNew} +import io.shiftleft.semanticcpg.language.types.structure.LocalTraversalNew import io.shiftleft.semanticcpg.langv2.Trav1 +import io.shiftleft.semanticcpg.langv2.types.structure.MethodTraversal object New { @@ -14,11 +15,11 @@ object New { new AstTraversalNew(trav) } - implicit def toMethodTraversalNew1[I <: nodes.Method, FT[_]](trav: I): MethodTraversalNew[I, Trav1, FT] = { - new MethodTraversalNew(trav: Trav1[I]) + implicit def toMethodTraversalNew1[I <: nodes.Method, FT[_]](trav: I): MethodTraversal[I, Trav1, FT] = { + new MethodTraversal(trav: Trav1[I]) } - implicit def toMethodTraversalNew2[I <: nodes.Method, IT[_], FT[_]](trav: IT[I]): MethodTraversalNew[I, IT, FT] = { - new MethodTraversalNew(trav) + implicit def toMethodTraversalNew2[I <: nodes.Method, IT[_], FT[_]](trav: IT[I]): MethodTraversal[I, IT, FT] = { + new MethodTraversal(trav) } implicit def toLocalTraversalNew1[I <: nodes.Local, FT[_]](trav: I): LocalTraversalNew[I, Trav1, FT] = { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala index 588cefec6..f915c6b84 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala @@ -55,13 +55,13 @@ class AstTraversalNew[I <: AstNode, IT[_], FT[_]](val trav: IT[I]) extends AnyVa * Nodes of the AST rooted in this node, including the node itself. * */ @Doc("All nodes of the abstract syntax tree") - def ast(implicit ops1: TravOps[IT, FT], ops2: TravNOps[FT]) = + def ast(implicit ops1: TravOps[IT, FT], ops2: TravOps[FT, FT]) = trav.rFlatMap(_._astOut.asScala.asInstanceOf[Iterator[I]], _.emit).cast[AstNode] /** * Traverse only to AST nodes that are expressions * */ - def isExpression(implicit ops1: TravOps[IT, FT], ops2: TravNOps[FT]) = + def isExpression(implicit ops1: TravOps[IT, FT], ops2: TravOps[FT, FT]) = trav.filter(_.isInstanceOf[Expression]).cast[Expression] } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala index 7e4b35cad..c62634943 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala @@ -47,7 +47,7 @@ object LocalReferencingIdentifiers { } class LocalTraversalNew[I <: nodes.Local, IT[_], FT[_]](val trav: IT[I]) extends AnyVal { - def referencingIdentifiers(implicit ops1: TravOps[IT, FT], ops2: TravNOps[FT]) = { + def referencingIdentifiers(implicit ops1: TravOps[IT, FT], ops2: TravOps[FT, FT]) = { trav .flatMap(_._refIn.asScala) .filter(_.label == NodeTypes.IDENTIFIER) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTraversal.scala index 12b5e4cea..bea14f76d 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTraversal.scala @@ -3,21 +3,9 @@ package io.shiftleft.semanticcpg.language.types.structure import io.shiftleft.codepropertygraph.generated._ import io.shiftleft.codepropertygraph.generated.nodes._ import io.shiftleft.semanticcpg.language._ -import io.shiftleft.semanticcpg.langv2._ import overflowdb._ import overflowdb.traversal.help.Doc import overflowdb.traversal.{Traversal, help} -import scala.jdk.CollectionConverters._ - -class MethodTraversalNew[I <: nodes.Method, IT[_], FT[_]](val trav: IT[I]) extends AnyVal { - def methodReturn()(implicit ops1: TravOps[IT, FT], ops2: TravNOps[FT]) = { - trav.map(_._astOut.asScala.collectFirst { case x: nodes.MethodReturn => x }.get) - } - - def methodParameters()(implicit ops1: TravOps[IT, FT], ops2: TravNOps[FT]) = { - trav.flatMap(_._astOut.asScala.collect { case x: nodes.MethodParameterIn => x }) - } -} /** * A method, function, or procedure diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala index 1d9dc98f0..f3b6d73d3 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala @@ -2,18 +2,12 @@ package io.shiftleft.semanticcpg package object langv2 { type Trav1[T] = T - type TravNOps[T[_]] = TravOps[T, T] implicit val trav1Ops = Trav1Ops implicit def iterableOps[IT[_] <: Iterable[_]] = IterableOps.asInstanceOf[TravOps[IT, IT]] - implicit def toAnyTraversal[I, IT[_], FT[_]](p: IT[I]): AnyTraversal[I, IT, FT] = { - new AnyTraversal(p) + implicit def toAnyTraversal[I, IT[_], FT[_]](trav: IT[I]): AnyTraversal[I, IT, FT] = { + new AnyTraversal(trav) } } -object foo { - implicit class MethoddParameterSingleExt(val node: Int) extends AnyVal { - def parameterB: Iterator[Int] = Iterator.single(2) - } -} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala new file mode 100644 index 000000000..274c863a6 --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala @@ -0,0 +1,34 @@ +package io.shiftleft.semanticcpg.langv2.types.structure + +import io.shiftleft.codepropertygraph.generated.nodes +import io.shiftleft.codepropertygraph.generated.nodes.Method +import io.shiftleft.semanticcpg.langv2._ +import overflowdb.traversal.help +import overflowdb.traversal.help.Doc + +import scala.jdk.CollectionConverters._ + + +/** + * A method, function, or procedure + * */ +@help.Traversal(elementType = classOf[Method]) +class MethodTraversal[I <: nodes.Method, IT[_], FT[_]](val trav: IT[I]) extends AnyVal { + + /** + * Traverse to parameters of the method + * */ + @Doc("All parameters") + def parameter(implicit ops1: TravOps[IT, FT]) = { + trav.flatMap(_._astOut.asScala.collect { case par: nodes.MethodParameterIn => par }) + } + + /** + * Traverse to formal return parameter + * */ + @Doc("All formal return parameters") + def methodReturn(implicit ops1: TravOps[IT, FT]) = { + trav.map(_._astOut.asScala.collectFirst { case ret: nodes.MethodReturn => ret }.get) + } + +} From 486b63837bed6ddf5552b399ed04d5f96242bee0 Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Wed, 17 Nov 2021 12:43:00 +0100 Subject: [PATCH 06/27] Improved collectAll performance. By using our own type class IsInstanceOfOps we now have the performance of a plain filter(_.isInstanceOf[...]). --- .../shiftleft/semanticcpg/langv2/AnyTraversal.scala | 13 +++---------- .../semanticcpg/langv2/IsInstanceOfOps.scala | 10 ++++++++++ .../io/shiftleft/semanticcpg/langv2/package.scala | 2 +- 3 files changed, 14 insertions(+), 11 deletions(-) create mode 100644 semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/IsInstanceOfOps.scala diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala index 24500fbd5..0aab1b7fa 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala @@ -29,16 +29,9 @@ class AnyTraversal[I, IT[_], FT[_]](val trav: IT[I]) extends AnyVal { trav.asInstanceOf[IT[A]] } - // This is about ten times slower than .filter(_.isInstanceOf[C]).cast[C] - // but thanks to JVM generics we cannot use isInstanceOf with type parameters. - // So use this method for convenience on a REPL and otherwise use filter. - def collectAll[C: ClassTag](implicit ops: TravOps[IT, FT]): FT[C] = { - ops - .filter(trav) { - case _: C => true - case _ => false - } - .cast[C] + // We dont use ClassTag and instead use our own IsInstanceOfOps for performance reasons. + def collectAll[T](implicit ops: TravOps[IT, FT], isInstanceOfOps: IsInstanceOfOps[T]): FT[T] = { + ops.filter(trav)(isInstanceOfOps).cast[T] } def rMap[O <: I](f: I => O, g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/IsInstanceOfOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/IsInstanceOfOps.scala new file mode 100644 index 000000000..783811a49 --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/IsInstanceOfOps.scala @@ -0,0 +1,10 @@ +package io.shiftleft.semanticcpg.langv2 + +import io.shiftleft.codepropertygraph.generated.nodes + +trait IsInstanceOfOps[T] extends Function1[Any, Boolean] + +trait InstanceOfOpsImplicits { + implicit val methodReturnCollectOps: IsInstanceOfOps[nodes.MethodReturn] = _.isInstanceOf[nodes.MethodReturn] + implicit val astNodeCollectOps: IsInstanceOfOps[nodes.AstNode] = _.isInstanceOf[nodes.AstNode] +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala index f3b6d73d3..ee3653b6b 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala @@ -1,6 +1,6 @@ package io.shiftleft.semanticcpg -package object langv2 { +package object langv2 extends InstanceOfOpsImplicits { type Trav1[T] = T implicit val trav1Ops = Trav1Ops From ea8e8d53cbabc9aa0ebd5f852c49b95f430b4d37 Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Wed, 17 Nov 2021 15:29:41 +0100 Subject: [PATCH 07/27] Add TravOps for Iterator. --- .../semanticcpg/langv2/IsInstanceOfOps.scala | 1 + .../semanticcpg/langv2/TravOps.scala | 23 +++++++++++++++++++ .../semanticcpg/langv2/package.scala | 5 ++++ 3 files changed, 29 insertions(+) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/IsInstanceOfOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/IsInstanceOfOps.scala index 783811a49..688e80425 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/IsInstanceOfOps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/IsInstanceOfOps.scala @@ -6,5 +6,6 @@ trait IsInstanceOfOps[T] extends Function1[Any, Boolean] trait InstanceOfOpsImplicits { implicit val methodReturnCollectOps: IsInstanceOfOps[nodes.MethodReturn] = _.isInstanceOf[nodes.MethodReturn] + implicit val methodParameterCollectOps: IsInstanceOfOps[nodes.MethodParameterIn] = _.isInstanceOf[nodes.MethodParameterIn] implicit val astNodeCollectOps: IsInstanceOfOps[nodes.AstNode] = _.isInstanceOf[nodes.AstNode] } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala index b939898e0..18ed9836d 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala @@ -64,3 +64,26 @@ object IterableOps extends TravOps[Iterable, Iterable] { trav.flatMap(f) } } + +object IteratorOps extends TravOps[Iterator, Iterator] { + + override def head[I](trav: Iterator[I]): I = { + trav.next() + } + + override def headOption[I](trav: Iterator[I]): Option[I] = { + trav.nextOption() + } + + override def map[I, O](trav: Iterator[I])(f: I => O): Iterator[O] = { + trav.map(f) + } + + override def filter[I](trav: Iterator[I])(f: I => Boolean): Iterator[I] = { + trav.filter(f) + } + + override def flatMap[I, O](trav: Iterator[I])(f: I => Iterator[O]): Iterator[O] = { + trav.flatMap(f) + } +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala index ee3653b6b..5135f29a9 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala @@ -4,7 +4,12 @@ package object langv2 extends InstanceOfOpsImplicits { type Trav1[T] = T implicit val trav1Ops = Trav1Ops + // All Iterable implementations coming from the scala standard library return an Iterable of + // the same runtime type as the input iterable as results of map, flatMap, filter etc.. + // We expect this to be true for all Iterabel implementations using iterableOps and thus cast + // to TravOps[IT, IT]. implicit def iterableOps[IT[_] <: Iterable[_]] = IterableOps.asInstanceOf[TravOps[IT, IT]] + implicit def iteratorOps[IT[_] <: Iterator[_]] = IteratorOps.asInstanceOf[TravOps[IT, Iterator]] implicit def toAnyTraversal[I, IT[_], FT[_]](trav: IT[I]): AnyTraversal[I, IT, FT] = { new AnyTraversal(trav) From c97d257537ecb8785316e145520193bacde1dac8 Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Thu, 18 Nov 2021 15:45:43 +0100 Subject: [PATCH 08/27] Tmp --- performance/src/main/scala/MyTests.scala | 9 +- .../src/main/scala/some/SomeDomain.scala | 14 +-- .../shiftleft/semanticcpg/language/New.scala | 12 +- .../generalizations/AstNodeTraversal.scala | 8 +- .../types/structure/LocalTraversal.scala | 10 +- .../semanticcpg/langv2/AnyTraversal.scala | 36 ++---- .../semanticcpg/langv2/IsInstanceOfOps.scala | 1 + .../semanticcpg/langv2/TravOps.scala | 114 ++++++++++++------ .../semanticcpg/langv2/package.scala | 33 +++-- .../types/structure/MethodTraversal.scala | 13 +- 10 files changed, 142 insertions(+), 108 deletions(-) diff --git a/performance/src/main/scala/MyTests.scala b/performance/src/main/scala/MyTests.scala index 9190aded6..7d5529483 100644 --- a/performance/src/main/scala/MyTests.scala +++ b/performance/src/main/scala/MyTests.scala @@ -1,7 +1,7 @@ package io.shiftleft.semanticcpg.language.types.expressions import io.shiftleft.codepropertygraph.generated.NodeTypes -import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, Method, MethodReturn, StoredNode} +import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, Expression, Method, MethodParameterIn, MethodReturn, StoredNode} import io.shiftleft.semanticcpg.testing.MockCpg import io.shiftleft.semanticcpg.language.MySteps._ import io.shiftleft.semanticcpg.language.ImportsV3._ @@ -104,7 +104,8 @@ class MyTestNew { @Benchmark def astTestNewV2(state: MyState) = { - toAstTraversalNew1(state.method).isExpression + val x: Option[Expression] = toAstTraversalNew1(state.method).isExpression + x } @Benchmark @@ -130,7 +131,9 @@ class MyTestNew { @Benchmark def syntheticIterableNew(state: MyState) = { //toSynth(state.d1:: Nil).toD2Multi - Iterable.single(state.d1).toD2 + val y: TravOps[Iterable] = toIt2Ops + val x: Iterable[D2] = Iterable.single(state.d1).toD2(toIt2Ops) + x } @Benchmark diff --git a/performance/src/main/scala/some/SomeDomain.scala b/performance/src/main/scala/some/SomeDomain.scala index 0377c0367..34198d10c 100644 --- a/performance/src/main/scala/some/SomeDomain.scala +++ b/performance/src/main/scala/some/SomeDomain.scala @@ -3,20 +3,20 @@ package some import io.shiftleft.semanticcpg.langv2._ object SomeDomain { - implicit def toSynth[FT[_]](p: D1): SynthExt[Trav1, FT] = { + implicit def toSynth(p: D1): SynthExt[Trav1] = { new SynthExt(p: Trav1[D1]) } - implicit def toSynth[IT[_], FT[_]](p: IT[D1]): SynthExt[IT, FT] = { + implicit def toSynth[IT[_]](p: IT[D1]): SynthExt[IT] = { new SynthExt(p) } - class SynthExt[IT[_], FT[_]](val trav: IT[D1]) extends AnyVal { - def toD2(implicit ops: TravOps[IT, FT]) = { - trav.map(_.x) + class SynthExt[IT[_]](val trav: IT[D1]) extends AnyVal { + def toD2(implicit ops: TravOps[IT]) = { + ops.oneToOne(trav)(_.x) } - def toD2Multi(implicit ops: TravOps[IT, FT]) = { - trav.flatMap(x => Iterator.single(x)) + def toD2Multi(implicit ops: TravOps[IT]) = { + ops.oneToMany(trav)(x => Iterator.single(x)) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala index 36c67255e..878e25d8a 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala @@ -8,25 +8,25 @@ import io.shiftleft.semanticcpg.langv2.types.structure.MethodTraversal object New { - implicit def toAstTraversalNew1[I <: nodes.AstNode, FT[_]](trav: I): AstTraversalNew[I, Trav1, FT] = { + implicit def toAstTraversalNew1[I <: nodes.AstNode](trav: I): AstTraversalNew[I, Trav1] = { new AstTraversalNew(trav: Trav1[I]) } - implicit def toAstTraversalNew2[I <: nodes.AstNode, IT[_], FT[_]](trav: IT[I]): AstTraversalNew[I, IT, FT] = { + implicit def toAstTraversalNew2[I <: nodes.AstNode, IT[_]](trav: IT[I]): AstTraversalNew[I, IT] = { new AstTraversalNew(trav) } - implicit def toMethodTraversalNew1[I <: nodes.Method, FT[_]](trav: I): MethodTraversal[I, Trav1, FT] = { + implicit def toMethodTraversalNew1[I <: nodes.Method](trav: I): MethodTraversal[I, Trav1] = { new MethodTraversal(trav: Trav1[I]) } - implicit def toMethodTraversalNew2[I <: nodes.Method, IT[_], FT[_]](trav: IT[I]): MethodTraversal[I, IT, FT] = { + implicit def toMethodTraversalNew2[I <: nodes.Method, IT[_]](trav: IT[I]): MethodTraversal[I, IT] = { new MethodTraversal(trav) } - implicit def toLocalTraversalNew1[I <: nodes.Local, FT[_]](trav: I): LocalTraversalNew[I, Trav1, FT] = { + implicit def toLocalTraversalNew1[I <: nodes.Local](trav: I): LocalTraversalNew[I, Trav1] = { new LocalTraversalNew(trav: Trav1[I]) } - implicit def toLocalTraversalNew2[I <: nodes.Local, IT[_], FT[_]](trav: IT[I]): LocalTraversalNew[I, IT, FT] = { + implicit def toLocalTraversalNew2[I <: nodes.Local, IT[_]](trav: IT[I]): LocalTraversalNew[I, IT] = { new LocalTraversalNew(trav) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala index f915c6b84..c8676d408 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala @@ -49,20 +49,20 @@ object AstNodeIsExpression { } } -class AstTraversalNew[I <: AstNode, IT[_], FT[_]](val trav: IT[I]) extends AnyVal { +class AstTraversalNew[I <: AstNode, IT[_]](val trav: IT[I]) extends AnyVal { /** * Nodes of the AST rooted in this node, including the node itself. * */ @Doc("All nodes of the abstract syntax tree") - def ast(implicit ops1: TravOps[IT, FT], ops2: TravOps[FT, FT]) = + def ast(implicit ops1: TravOps[IT]) = trav.rFlatMap(_._astOut.asScala.asInstanceOf[Iterator[I]], _.emit).cast[AstNode] /** * Traverse only to AST nodes that are expressions * */ - def isExpression(implicit ops1: TravOps[IT, FT], ops2: TravOps[FT, FT]) = - trav.filter(_.isInstanceOf[Expression]).cast[Expression] + def isExpression(implicit ops1: TravOps[IT]) = + trav.collectAll[Expression] } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala index c62634943..73e1c4e6a 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala @@ -46,12 +46,10 @@ object LocalReferencingIdentifiers { } } -class LocalTraversalNew[I <: nodes.Local, IT[_], FT[_]](val trav: IT[I]) extends AnyVal { - def referencingIdentifiers(implicit ops1: TravOps[IT, FT], ops2: TravOps[FT, FT]) = { - trav - .flatMap(_._refIn.asScala) - .filter(_.label == NodeTypes.IDENTIFIER) - .cast[Identifier] +class LocalTraversalNew[I <: nodes.Local, IT[_]](val trav: IT[I]) extends AnyVal { + def referencingIdentifiers(implicit ops1: TravOps[IT]): ops1.CCOneToMany[Identifier] = { + ops1.oneToMany(trav)(_._refIn.asScala.filter(_.label == NodeTypes.IDENTIFIER) + .cast[Identifier]) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala index 0aab1b7fa..297770bf3 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala @@ -2,53 +2,33 @@ package io.shiftleft.semanticcpg.langv2 import overflowdb.traversal.RepeatStepIterator -import scala.reflect.ClassTag -class AnyTraversal[I, IT[_], FT[_]](val trav: IT[I]) extends AnyVal { - def head(implicit ops: TravOps[IT, FT]): I = { - ops.head(trav) - } - - def headOption(implicit ops: TravOps[IT, FT]): Option[I] = { - ops.headOption(trav) - } - - def map[O](f: I => O)(implicit ops: TravOps[IT, FT]): IT[O] = { - ops.map(trav)(f) - } - - def filter(f: I => Boolean)(implicit ops: TravOps[IT, FT]): FT[I] = { - ops.filter(trav)(f) - } - - def flatMap[O](f: I => Iterator[O])(implicit ops: TravOps[IT, FT]): FT[O] = { - ops.flatMap(trav)(f) - } +class AnyTraversal[I, IT[_]](val trav: IT[I]) extends AnyVal { def cast[A]: IT[A] = { trav.asInstanceOf[IT[A]] } // We dont use ClassTag and instead use our own IsInstanceOfOps for performance reasons. - def collectAll[T](implicit ops: TravOps[IT, FT], isInstanceOfOps: IsInstanceOfOps[T]): FT[T] = { - ops.filter(trav)(isInstanceOfOps).cast[T] + def collectAll[T](implicit ops: TravOps[IT], isInstanceOfOps: IsInstanceOfOps[T]): ops.CCOneToBoolean[T] = { + ops.oneToBoolean(trav)(isInstanceOfOps).cast[T] } def rMap[O <: I](f: I => O, g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[IT, FT]): FT[I] = { + implicit ops: TravOps[IT]): ops.CCOneToMany[I] = { val behaviour = g(new RepeatBehaviourBuilder()).build - ops.flatMap(trav)(i => new RepeatStepIterator(i, f.andThen(Iterator.single), behaviour)) + ops.oneToMany(trav)(i => new RepeatStepIterator(i, f.andThen(Iterator.single), behaviour)) } def rFlatMap[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[IT, FT]): FT[I] = { + implicit ops: TravOps[IT]): ops.CCOneToMany[I] = { val behaviour = g(new RepeatBehaviourBuilder()).build - ops.flatMap(trav)(i => new RepeatStepIterator(i, f, behaviour)) + ops.oneToMany(trav)(i => new RepeatStepIterator(i, f, behaviour)) } @deprecated("Use rFlatMap instead") def repeat[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[IT, FT]): FT[I] = { + implicit ops: TravOps[IT]): ops.CCOneToMany[I] = { rFlatMap(f, g) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/IsInstanceOfOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/IsInstanceOfOps.scala index 688e80425..f7db7977c 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/IsInstanceOfOps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/IsInstanceOfOps.scala @@ -8,4 +8,5 @@ trait InstanceOfOpsImplicits { implicit val methodReturnCollectOps: IsInstanceOfOps[nodes.MethodReturn] = _.isInstanceOf[nodes.MethodReturn] implicit val methodParameterCollectOps: IsInstanceOfOps[nodes.MethodParameterIn] = _.isInstanceOf[nodes.MethodParameterIn] implicit val astNodeCollectOps: IsInstanceOfOps[nodes.AstNode] = _.isInstanceOf[nodes.AstNode] + implicit val expressionCollectOps: IsInstanceOfOps[nodes.Expression] = _.isInstanceOf[nodes.Expression] } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala index 18ed9836d..9c5adc889 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala @@ -1,89 +1,125 @@ package io.shiftleft.semanticcpg.langv2 +import scala.collection.IterableOnceOps + /** Traversal operations trait. * Implementations of this trait handle the basic traversal operations for the * different kind of input traversal types. + * * @tparam InputTraversal Input traversal type. Abbreviated in other generics as IT. * @tparam FlatMapTraversal Flat map operation output traversal type. * Abbreviated in other generics as FT. */ -trait TravOps[InputTraversal[_], FlatMapTraversal[_]] { - def head[I](trav: InputTraversal[I]): I - def headOption[I](trav: InputTraversal[I]): Option[I] - def map[I, O](trav: InputTraversal[I])(f: I => O): InputTraversal[O] - def filter[I](trav: InputTraversal[I])(f: I => Boolean): FlatMapTraversal[I] - def flatMap[I, O](trav: InputTraversal[I])(f: I => Iterator[O]): FlatMapTraversal[O] +trait TravOps[InputTraversal[_]] { + type CCOneToOne[_] + type CCOneToOption[_] + type CCOneToBoolean[_] + type CCOneToMany[_] + + def oneToOne[I, O](trav: InputTraversal[I])(f: I => O): CCOneToOne[O] + def oneToOption[I, O](trav: InputTraversal[I])(f: I => Option[O]): CCOneToOption[O] + def oneToBoolean[I](trav: InputTraversal[I])(f: I => Boolean): CCOneToBoolean[I] + def oneToMany[I, O](trav: InputTraversal[I])(f: I => Iterator[O]): CCOneToMany[O] } -object Trav1Ops extends TravOps[Trav1, Iterable] { - - override def head[I](trav: Trav1[I]): I = { - trav - } +object Trav1Ops extends TravOps[Trav1] { + override type CCOneToOne[T] = T + override type CCOneToOption[T] = Option[T] + override type CCOneToBoolean[T] = Option[T] + override type CCOneToMany[T] = Seq[T] - override def headOption[I](trav: Trav1[I]): Option[I] = { - Some(trav) + override def oneToOne[I, O](trav: Trav1[I])(f: I => O): CCOneToOne[O] = { + f(trav) } - override def map[I, O](trav: Trav1[I])(f: I => O): Trav1[O] = { + override def oneToOption[I, O](trav: Trav1[I])(f: I => Option[O]): CCOneToOption[O] = { f(trav) } - override def filter[I](trav: Trav1[I])(f: I => Boolean): Iterable[I] = { + override def oneToBoolean[I](trav: Trav1[I])(f: I => Boolean): CCOneToBoolean[I] = { if (f(trav)) { - trav :: Nil + Some(trav) } else { - Nil + None } } - override def flatMap[I, O](trav: Trav1[I])(f: I => Iterator[O]): Iterable[O] = { - Iterable.from(f(trav)) + override def oneToMany[I, O](trav: Trav1[I])(f: I => Iterator[O]): CCOneToMany[O] = { + Seq.from(f(trav)) } } -object IterableOps extends TravOps[Iterable, Iterable] { - - override def head[I](trav: Iterable[I]): I = { - trav.head - } +object OptionOps extends TravOps[Option] { + override type CCOneToOne[T] = Option[T] + override type CCOneToOption[T] = Option[T] + override type CCOneToBoolean[T] = Option[T] + override type CCOneToMany[T] = Seq[T] - override def headOption[I](trav: Iterable[I]): Option[I] = { - trav.headOption + override def oneToOne[I, O](trav: Option[I])(f: I => O): CCOneToOne[O] = { + trav.map(f) } - override def map[I, O](trav: Iterable[I])(f: I => O): Iterable[O] = { - trav.map(f) + override def oneToOption[I, O](trav: Option[I])(f: I => Option[O]): CCOneToOption[O] = { + trav.flatMap(f) } - override def filter[I](trav: Iterable[I])(f: I => Boolean): Iterable[I] = { + override def oneToBoolean[I](trav: Option[I])(f: I => Boolean): CCOneToBoolean[I] = { trav.filter(f) } - override def flatMap[I, O](trav: Iterable[I])(f: I => Iterator[O]): Iterable[O] = { - trav.flatMap(f) + override def oneToMany[I, O](trav: Option[I])(f: I => Iterator[O]): CCOneToMany[O] = { + trav match { + case Some(t) => + Seq.from(f(t)) + case None => + Seq.empty + } } } -object IteratorOps extends TravOps[Iterator, Iterator] { +class IterableOps[Col[A] <: Iterable[A] ] extends TravOps[Col] { + override type CCOneToOne[T] = Col[T] + override type CCOneToOption[T] = Col[T] + override type CCOneToBoolean[T] = Col[T] + override type CCOneToMany[T] = Col[T] + + override def oneToOne[I, O](trav: Col[I])(f: I => O): CCOneToOne[O] = { + trav.map(f).asInstanceOf[Col[O]] + } + + override def oneToOption[I, O](trav: Col[I])(f: I => Option[O]): CCOneToOption[O] = { + trav.flatMap(f).asInstanceOf[Col[O]] + } - override def head[I](trav: Iterator[I]): I = { - trav.next() + override def oneToBoolean[I](trav: Col[I])(f: I => Boolean): CCOneToBoolean[I] = { + trav.filter(f).asInstanceOf[Col[I]] } - override def headOption[I](trav: Iterator[I]): Option[I] = { - trav.nextOption() + override def oneToMany[I, O](trav: Col[I])(f: I => Iterator[O]): CCOneToMany[O] = { + trav.flatMap(f).asInstanceOf[Col[O]] } +} + +//class IterableOnceOpsOps[CC[_], C](unused: IterableOnceOps[_, CC, C]) extends TravOps[({type X[B] = IterableOnceOps[B, CC, C]})#X] { +class IterableOnceOpsOps[CC[_], C] extends TravOps[({type X[B] = IterableOnceOps[B, CC, C]})#X] { + override type CCOneToOne[T] = CC[T] + override type CCOneToOption[T] = CC[T] + override type CCOneToBoolean[T] = C + override type CCOneToMany[T] = CC[T] - override def map[I, O](trav: Iterator[I])(f: I => O): Iterator[O] = { + override def oneToOne[I, O](trav: IterableOnceOps[I, CC, C])(f: I => O): CCOneToOne[O] = { trav.map(f) } - override def filter[I](trav: Iterator[I])(f: I => Boolean): Iterator[I] = { + override def oneToOption[I, O](trav: IterableOnceOps[I, CC, C])(f: I => Option[O]): CCOneToOption[O] = { + trav.flatMap(f) + } + + override def oneToBoolean[I](trav: IterableOnceOps[I, CC, C])(f: I => Boolean): CCOneToBoolean[I] = { trav.filter(f) } - override def flatMap[I, O](trav: Iterator[I])(f: I => Iterator[O]): Iterator[O] = { + override def oneToMany[I, O](trav: IterableOnceOps[I, CC, C])(f: I => Iterator[O]): CCOneToMany[O] = { trav.flatMap(f) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala index 5135f29a9..076d30e27 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala @@ -1,17 +1,34 @@ package io.shiftleft.semanticcpg +import scala.collection.{IterableFactory, IterableOnceOps} + package object langv2 extends InstanceOfOpsImplicits { type Trav1[T] = T implicit val trav1Ops = Trav1Ops - // All Iterable implementations coming from the scala standard library return an Iterable of - // the same runtime type as the input iterable as results of map, flatMap, filter etc.. - // We expect this to be true for all Iterabel implementations using iterableOps and thus cast - // to TravOps[IT, IT]. - implicit def iterableOps[IT[_] <: Iterable[_]] = IterableOps.asInstanceOf[TravOps[IT, IT]] - implicit def iteratorOps[IT[_] <: Iterator[_]] = IteratorOps.asInstanceOf[TravOps[IT, Iterator]] - - implicit def toAnyTraversal[I, IT[_], FT[_]](trav: IT[I]): AnyTraversal[I, IT, FT] = { + + + private val it2Ops = new IterableOps() + implicit def toIt2Ops[Col[A] <: Iterable[A]] = it2Ops.asInstanceOf[IterableOps[Col]] + + private val iterableOnceOpsOps = new IterableOnceOpsOps() + //implicit def toIterOnceOps[CC[_], C]: TravOps[({type X[+B] = IterableOnceOps[B, CC, C]})#X] = { + // iterableOnceOpsOps.asInstanceOf[TravOps[({type X[+B] = IterableOnceOps[B, CC, C]})#X]] + //} + + // This works if u provide CC and C eplicitly as [Iterable, Iterable[Any]] + implicit def toIterOnceOps[CC[_], C]: IterableOnceOpsOps[CC, C] = { + iterableOnceOpsOps.asInstanceOf[IterableOnceOpsOps[CC, C]] + } + + // Iterable[A] = IterableOnceOps[A, Iterable, Iterable[A]] + // io.shiftleft.semanticcpg.langv2.IterableOnceOpsOps[[+A]Iterable[A],Iterable[Any]] + + + + // implicit val iterOps = new IterableOnceOpsOps(Iterable.empty[Any]) + + implicit def toAnyTraversal[I, IT[_]](trav: IT[I]): AnyTraversal[I, IT] = { new AnyTraversal(trav) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala index 274c863a6..8820f76b8 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala @@ -1,7 +1,6 @@ package io.shiftleft.semanticcpg.langv2.types.structure import io.shiftleft.codepropertygraph.generated.nodes -import io.shiftleft.codepropertygraph.generated.nodes.Method import io.shiftleft.semanticcpg.langv2._ import overflowdb.traversal.help import overflowdb.traversal.help.Doc @@ -12,23 +11,23 @@ import scala.jdk.CollectionConverters._ /** * A method, function, or procedure * */ -@help.Traversal(elementType = classOf[Method]) -class MethodTraversal[I <: nodes.Method, IT[_], FT[_]](val trav: IT[I]) extends AnyVal { +@help.Traversal(elementType = classOf[nodes.Method]) +class MethodTraversal[I <: nodes.Method, IT[_]](val trav: IT[I]) extends AnyVal { /** * Traverse to parameters of the method * */ @Doc("All parameters") - def parameter(implicit ops1: TravOps[IT, FT]) = { - trav.flatMap(_._astOut.asScala.collect { case par: nodes.MethodParameterIn => par }) + def parameter(implicit ops1: TravOps[IT]) = { + ops1.oneToMany(trav)(_._astOut.asScala.collect { case par: nodes.MethodParameterIn => par }) } /** * Traverse to formal return parameter * */ @Doc("All formal return parameters") - def methodReturn(implicit ops1: TravOps[IT, FT]) = { - trav.map(_._astOut.asScala.collectFirst { case ret: nodes.MethodReturn => ret }.get) + def methodReturn(implicit ops1: TravOps[IT]) = { + ops1.oneToOne(trav)(_._astOut.asScala.collectFirst { case ret: nodes.MethodReturn => ret }.get) } } From 98bdeef230f3475ba6068da0fd9f4a2629d029c1 Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Thu, 18 Nov 2021 20:53:21 +0100 Subject: [PATCH 09/27] Working --- performance/src/main/scala/MyTests.scala | 5 +- .../src/main/scala/some/SomeDomain.scala | 25 +++-- .../shiftleft/semanticcpg/language/New.scala | 47 +++++--- .../generalizations/AstNodeTraversal.scala | 7 +- .../types/structure/LocalTraversal.scala | 4 +- .../semanticcpg/langv2/AnyTraversal.scala | 16 +-- .../semanticcpg/langv2/TravOps.scala | 102 ++++++++---------- .../semanticcpg/langv2/package.scala | 52 ++++++--- .../types/structure/MethodTraversal.scala | 6 +- 9 files changed, 153 insertions(+), 111 deletions(-) diff --git a/performance/src/main/scala/MyTests.scala b/performance/src/main/scala/MyTests.scala index 7d5529483..76ebe31b4 100644 --- a/performance/src/main/scala/MyTests.scala +++ b/performance/src/main/scala/MyTests.scala @@ -131,8 +131,9 @@ class MyTestNew { @Benchmark def syntheticIterableNew(state: MyState) = { //toSynth(state.d1:: Nil).toD2Multi - val y: TravOps[Iterable] = toIt2Ops - val x: Iterable[D2] = Iterable.single(state.d1).toD2(toIt2Ops) + //val y: TravOps[Iterable, IterableTypes[Iterable, Iterable[Any]]] = toIt2Ops + val c = toSynth3(Array(state.d1).view.slice(1,2)).toD2 + val x: List[D2] = List(state.d1).toD2(toIt2Ops) x } diff --git a/performance/src/main/scala/some/SomeDomain.scala b/performance/src/main/scala/some/SomeDomain.scala index 34198d10c..33afc468d 100644 --- a/performance/src/main/scala/some/SomeDomain.scala +++ b/performance/src/main/scala/some/SomeDomain.scala @@ -2,20 +2,29 @@ package some import io.shiftleft.semanticcpg.langv2._ +import scala.collection.IterableOnceOps + object SomeDomain { - implicit def toSynth(p: D1): SynthExt[Trav1] = { - new SynthExt(p: Trav1[D1]) + implicit def toSynth1(p: D1) = { + new SynthExt[Trav1, SingleTypes.type](p: Trav1[D1]) } - - implicit def toSynth[IT[_]](p: IT[D1]): SynthExt[IT] = { - new SynthExt(p) + implicit def toSynth2(trav: Option[D1]) = { + new SynthExt[Option, OptionTypes.type](trav) } + implicit def toSynth3[CC[_], C](trav: IterableOnceOps[D1, CC, C]) = { + //new AstTraversalNew[I, ({type X[B] = IterableOnceOps[B, CC, C]})#X, CC, CC, ({type X[B] = C})#X, CC](trav) + new SynthExt[({type X[B] = IterableOnceOps[B, CC, C]})#X, IterableTypes[CC, C]](trav) + } + + //implicit def toSynth[IT[_]](p: IT[D1]): SynthExt[IT] = { + // new SynthExt(p) + //} - class SynthExt[IT[_]](val trav: IT[D1]) extends AnyVal { - def toD2(implicit ops: TravOps[IT]) = { + class SynthExt[IT[_], TM <: TypeMultiplexer](val trav: IT[D1]) extends AnyVal { + def toD2(implicit ops: TravOps[IT, TM]) = { ops.oneToOne(trav)(_.x) } - def toD2Multi(implicit ops: TravOps[IT]) = { + def toD2Multi(implicit ops: TravOps[IT, TM]) = { ops.oneToMany(trav)(x => Iterator.single(x)) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala index 878e25d8a..d0e66c97c 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala @@ -3,30 +3,51 @@ package io.shiftleft.semanticcpg.language import io.shiftleft.codepropertygraph.generated.nodes import io.shiftleft.semanticcpg.language.types.expressions.generalizations.AstTraversalNew import io.shiftleft.semanticcpg.language.types.structure.LocalTraversalNew -import io.shiftleft.semanticcpg.langv2.Trav1 +import io.shiftleft.semanticcpg.langv2.{IterableTypes, OptionTypes, SingleTypes, Trav1} import io.shiftleft.semanticcpg.langv2.types.structure.MethodTraversal +import scala.collection.IterableOnceOps + object New { - implicit def toAstTraversalNew1[I <: nodes.AstNode](trav: I): AstTraversalNew[I, Trav1] = { - new AstTraversalNew(trav: Trav1[I]) + implicit def toAstTraversalNew1[I <: nodes.AstNode](trav: I) = { + new AstTraversalNew[I, Trav1, SingleTypes.type](trav: Trav1[I]) + } + implicit def toAstTraversalNew2[I <: nodes.AstNode](trav: Option[I]) = { + new AstTraversalNew[I, Option, OptionTypes.type](trav) } - implicit def toAstTraversalNew2[I <: nodes.AstNode, IT[_]](trav: IT[I]): AstTraversalNew[I, IT] = { - new AstTraversalNew(trav) + //implicit def toAstTraversalNew2[I <: nodes.AstNode, IT[_]](trav: IT[I]): AstTraversalNew[I, IT] = { + // new AstTraversalNew(trav) + //} + implicit def toAstTraversalNew3[I <: nodes.AstNode, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { + //new AstTraversalNew[I, ({type X[B] = IterableOnceOps[B, CC, C]})#X, CC, CC, ({type X[B] = C})#X, CC](trav) + new AstTraversalNew[I, ({type X[B] = IterableOnceOps[B, CC, C]})#X, IterableTypes[CC, C]](trav) } - implicit def toMethodTraversalNew1[I <: nodes.Method](trav: I): MethodTraversal[I, Trav1] = { - new MethodTraversal(trav: Trav1[I]) + implicit def toMethodTraversalNew1[I <: nodes.Method](trav: I) = { + new MethodTraversal[I, Trav1, SingleTypes.type](trav: Trav1[I]) } - implicit def toMethodTraversalNew2[I <: nodes.Method, IT[_]](trav: IT[I]): MethodTraversal[I, IT] = { - new MethodTraversal(trav) + implicit def toMethodTraversalNew2[I <: nodes.Method](trav: Option[I]) = { + new MethodTraversal[I, Option, OptionTypes.type](trav) + } + //implicit def toMethodTraversalNew2[I <: nodes.Method, IT[_]](trav: IT[I]): MethodTraversal[I, IT] = { + // new MethodTraversal(trav) + //} + implicit def toMethodTraversalNew3[I <: nodes.Method, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { + new MethodTraversal[I, ({type X[B] = IterableOnceOps[B, CC, C]})#X, IterableTypes[CC, C]](trav) } - implicit def toLocalTraversalNew1[I <: nodes.Local](trav: I): LocalTraversalNew[I, Trav1] = { - new LocalTraversalNew(trav: Trav1[I]) + implicit def toLocalTraversalNew1[I <: nodes.Local](trav: I) = { + new LocalTraversalNew[I, Trav1, SingleTypes.type](trav: Trav1[I]) + } + implicit def toLocalTraversalNew2[I <: nodes.Local](trav: Option[I]) = { + new LocalTraversalNew[I, Option, OptionTypes.type](trav) } - implicit def toLocalTraversalNew2[I <: nodes.Local, IT[_]](trav: IT[I]): LocalTraversalNew[I, IT] = { - new LocalTraversalNew(trav) + //implicit def toLocalTraversalNew2[I <: nodes.Local, IT[_]](trav: IT[I]): LocalTraversalNew[I, IT] = { + // new LocalTraversalNew(trav) + //} + implicit def toLocalTraversalNew3[I <: nodes.Local, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { + new LocalTraversalNew[I, ({type X[B] = IterableOnceOps[B, CC, C]})#X, IterableTypes[CC, C]](trav) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala index c8676d408..f3135ada7 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala @@ -49,19 +49,20 @@ object AstNodeIsExpression { } } -class AstTraversalNew[I <: AstNode, IT[_]](val trav: IT[I]) extends AnyVal { +class AstTraversalNew[I <: AstNode, IT[_], TM <: TypeMultiplexer](val trav: IT[I]) + extends AnyVal { /** * Nodes of the AST rooted in this node, including the node itself. * */ @Doc("All nodes of the abstract syntax tree") - def ast(implicit ops1: TravOps[IT]) = + def ast(implicit ops1: TravOps[IT, TM]) = trav.rFlatMap(_._astOut.asScala.asInstanceOf[Iterator[I]], _.emit).cast[AstNode] /** * Traverse only to AST nodes that are expressions * */ - def isExpression(implicit ops1: TravOps[IT]) = + def isExpression(implicit ops1: TravOps[IT, TM]) = trav.collectAll[Expression] } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala index 73e1c4e6a..772b30b71 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala @@ -46,8 +46,8 @@ object LocalReferencingIdentifiers { } } -class LocalTraversalNew[I <: nodes.Local, IT[_]](val trav: IT[I]) extends AnyVal { - def referencingIdentifiers(implicit ops1: TravOps[IT]): ops1.CCOneToMany[Identifier] = { +class LocalTraversalNew[I <: nodes.Local, IT[_], TM <: TypeMultiplexer](val trav: IT[I]) extends AnyVal { + def referencingIdentifiers(implicit ops1: TravOps[IT, TM]): TM#CCOneToMany[Identifier] = { ops1.oneToMany(trav)(_._refIn.asScala.filter(_.label == NodeTypes.IDENTIFIER) .cast[Identifier]) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala index 297770bf3..dc6c86deb 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala @@ -3,32 +3,32 @@ package io.shiftleft.semanticcpg.langv2 import overflowdb.traversal.RepeatStepIterator -class AnyTraversal[I, IT[_]](val trav: IT[I]) extends AnyVal { +class AnyTraversal[I, IT[_], TM <: TypeMultiplexer](val trav: IT[I]) extends AnyVal { - def cast[A]: IT[A] = { - trav.asInstanceOf[IT[A]] + def cast[A]: TM#CCOneToOne[A] = { + trav.asInstanceOf[TM#CCOneToOne[A]] } // We dont use ClassTag and instead use our own IsInstanceOfOps for performance reasons. - def collectAll[T](implicit ops: TravOps[IT], isInstanceOfOps: IsInstanceOfOps[T]): ops.CCOneToBoolean[T] = { - ops.oneToBoolean(trav)(isInstanceOfOps).cast[T] + def collectAll[T](implicit ops: TravOps[IT, TM], isInstanceOfOps: IsInstanceOfOps[T]): TM#CCOneToBoolean[T] = { + ops.oneToBoolean(trav)(isInstanceOfOps).asInstanceOf[TM#CCOneToBoolean[T]] } def rMap[O <: I](f: I => O, g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[IT]): ops.CCOneToMany[I] = { + implicit ops: TravOps[IT, TM]): TM#CCOneToMany[I] = { val behaviour = g(new RepeatBehaviourBuilder()).build ops.oneToMany(trav)(i => new RepeatStepIterator(i, f.andThen(Iterator.single), behaviour)) } def rFlatMap[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[IT]): ops.CCOneToMany[I] = { + implicit ops: TravOps[IT, TM]): TM#CCOneToMany[I] = { val behaviour = g(new RepeatBehaviourBuilder()).build ops.oneToMany(trav)(i => new RepeatStepIterator(i, f, behaviour)) } @deprecated("Use rFlatMap instead") def repeat[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[IT]): ops.CCOneToMany[I] = { + implicit ops: TravOps[IT, TM]): TM#CCOneToMany[I] = { rFlatMap(f, g) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala index 9c5adc889..a96f9e4ba 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala @@ -10,33 +10,25 @@ import scala.collection.IterableOnceOps * @tparam FlatMapTraversal Flat map operation output traversal type. * Abbreviated in other generics as FT. */ -trait TravOps[InputTraversal[_]] { - type CCOneToOne[_] - type CCOneToOption[_] - type CCOneToBoolean[_] - type CCOneToMany[_] - - def oneToOne[I, O](trav: InputTraversal[I])(f: I => O): CCOneToOne[O] - def oneToOption[I, O](trav: InputTraversal[I])(f: I => Option[O]): CCOneToOption[O] - def oneToBoolean[I](trav: InputTraversal[I])(f: I => Boolean): CCOneToBoolean[I] - def oneToMany[I, O](trav: InputTraversal[I])(f: I => Iterator[O]): CCOneToMany[O] +trait TravOps[InputTraversal[_], TM <: TypeMultiplexer] { + type tm = TM + def oneToOne[I, O](trav: InputTraversal[I])(f: I => O): TM#CCOneToOne[O] + def oneToOption[I, O](trav: InputTraversal[I])(f: I => Option[O]): TM#CCOneToOption[O] + def oneToBoolean[I](trav: InputTraversal[I])(f: I => Boolean): TM#CCOneToBoolean[I] + def oneToMany[I, O](trav: InputTraversal[I])(f: I => Iterator[O]): TM#CCOneToMany[O] } -object Trav1Ops extends TravOps[Trav1] { - override type CCOneToOne[T] = T - override type CCOneToOption[T] = Option[T] - override type CCOneToBoolean[T] = Option[T] - override type CCOneToMany[T] = Seq[T] +object Trav1Ops extends TravOps[Trav1, SingleTypes.type] { - override def oneToOne[I, O](trav: Trav1[I])(f: I => O): CCOneToOne[O] = { + override def oneToOne[I, O](trav: Trav1[I])(f: I => O): this.type#tm#CCOneToOne[O] = { f(trav) } - override def oneToOption[I, O](trav: Trav1[I])(f: I => Option[O]): CCOneToOption[O] = { + override def oneToOption[I, O](trav: Trav1[I])(f: I => Option[O]): this.type#tm#CCOneToOption[O] = { f(trav) } - override def oneToBoolean[I](trav: Trav1[I])(f: I => Boolean): CCOneToBoolean[I] = { + override def oneToBoolean[I](trav: Trav1[I])(f: I => Boolean): this.type#tm#CCOneToBoolean[I] = { if (f(trav)) { Some(trav) } else { @@ -44,30 +36,26 @@ object Trav1Ops extends TravOps[Trav1] { } } - override def oneToMany[I, O](trav: Trav1[I])(f: I => Iterator[O]): CCOneToMany[O] = { + override def oneToMany[I, O](trav: Trav1[I])(f: I => Iterator[O]): this.type#tm#CCOneToMany[O] = { Seq.from(f(trav)) } } -object OptionOps extends TravOps[Option] { - override type CCOneToOne[T] = Option[T] - override type CCOneToOption[T] = Option[T] - override type CCOneToBoolean[T] = Option[T] - override type CCOneToMany[T] = Seq[T] +object OptionOps extends TravOps[Option, OptionTypes.type ] { - override def oneToOne[I, O](trav: Option[I])(f: I => O): CCOneToOne[O] = { + override def oneToOne[I, O](trav: Option[I])(f: I => O): this.type#tm#CCOneToOne[O] = { trav.map(f) } - override def oneToOption[I, O](trav: Option[I])(f: I => Option[O]): CCOneToOption[O] = { + override def oneToOption[I, O](trav: Option[I])(f: I => Option[O]): this.type#tm#CCOneToOption[O] = { trav.flatMap(f) } - override def oneToBoolean[I](trav: Option[I])(f: I => Boolean): CCOneToBoolean[I] = { + override def oneToBoolean[I](trav: Option[I])(f: I => Boolean): this.type#tm#CCOneToBoolean[I] = { trav.filter(f) } - override def oneToMany[I, O](trav: Option[I])(f: I => Iterator[O]): CCOneToMany[O] = { + override def oneToMany[I, O](trav: Option[I])(f: I => Iterator[O]): this.type#tm#CCOneToMany[O] = { trav match { case Some(t) => Seq.from(f(t)) @@ -77,49 +65,45 @@ object OptionOps extends TravOps[Option] { } } -class IterableOps[Col[A] <: Iterable[A] ] extends TravOps[Col] { - override type CCOneToOne[T] = Col[T] - override type CCOneToOption[T] = Col[T] - override type CCOneToBoolean[T] = Col[T] - override type CCOneToMany[T] = Col[T] - - override def oneToOne[I, O](trav: Col[I])(f: I => O): CCOneToOne[O] = { - trav.map(f).asInstanceOf[Col[O]] - } - - override def oneToOption[I, O](trav: Col[I])(f: I => Option[O]): CCOneToOption[O] = { - trav.flatMap(f).asInstanceOf[Col[O]] - } - - override def oneToBoolean[I](trav: Col[I])(f: I => Boolean): CCOneToBoolean[I] = { - trav.filter(f).asInstanceOf[Col[I]] - } - - override def oneToMany[I, O](trav: Col[I])(f: I => Iterator[O]): CCOneToMany[O] = { - trav.flatMap(f).asInstanceOf[Col[O]] - } -} +//class IterableOps[Col[A] <: Iterable[A] ] extends TravOps[Col] { +// override type CCOneToOne[T] = Col[T] +// override type CCOneToOption[T] = Col[T] +// override type CCOneToBoolean[T] = Col[T] +// override type CCOneToMany[T] = Col[T] +// +// override def oneToOne[I, O](trav: Col[I])(f: I => O): CCOneToOne[O] = { +// trav.map(f).asInstanceOf[Col[O]] +// } +// +// override def oneToOption[I, O](trav: Col[I])(f: I => Option[O]): CCOneToOption[O] = { +// trav.flatMap(f).asInstanceOf[Col[O]] +// } +// +// override def oneToBoolean[I](trav: Col[I])(f: I => Boolean): CCOneToBoolean[I] = { +// trav.filter(f).asInstanceOf[Col[I]] +// } +// +// override def oneToMany[I, O](trav: Col[I])(f: I => Iterator[O]): CCOneToMany[O] = { +// trav.flatMap(f).asInstanceOf[Col[O]] +// } +//} //class IterableOnceOpsOps[CC[_], C](unused: IterableOnceOps[_, CC, C]) extends TravOps[({type X[B] = IterableOnceOps[B, CC, C]})#X] { -class IterableOnceOpsOps[CC[_], C] extends TravOps[({type X[B] = IterableOnceOps[B, CC, C]})#X] { - override type CCOneToOne[T] = CC[T] - override type CCOneToOption[T] = CC[T] - override type CCOneToBoolean[T] = C - override type CCOneToMany[T] = CC[T] +class IterableOnceOpsOps[CC[_], C] extends TravOps[({type X[B] = IterableOnceOps[B, CC, C]})#X, IterableTypes[CC, C] ] { - override def oneToOne[I, O](trav: IterableOnceOps[I, CC, C])(f: I => O): CCOneToOne[O] = { + override def oneToOne[I, O](trav: IterableOnceOps[I, CC, C])(f: I => O): this.type#tm#CCOneToOne[O] = { trav.map(f) } - override def oneToOption[I, O](trav: IterableOnceOps[I, CC, C])(f: I => Option[O]): CCOneToOption[O] = { + override def oneToOption[I, O](trav: IterableOnceOps[I, CC, C])(f: I => Option[O]): this.type#tm#CCOneToOption[O] = { trav.flatMap(f) } - override def oneToBoolean[I](trav: IterableOnceOps[I, CC, C])(f: I => Boolean): CCOneToBoolean[I] = { + override def oneToBoolean[I](trav: IterableOnceOps[I, CC, C])(f: I => Boolean): this.type#tm#CCOneToBoolean[I] = { trav.filter(f) } - override def oneToMany[I, O](trav: IterableOnceOps[I, CC, C])(f: I => Iterator[O]): CCOneToMany[O] = { + override def oneToMany[I, O](trav: IterableOnceOps[I, CC, C])(f: I => Iterator[O]): this.type#tm#CCOneToMany[O] = { trav.flatMap(f) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala index 076d30e27..5d2bdfe35 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala @@ -5,22 +5,41 @@ import scala.collection.{IterableFactory, IterableOnceOps} package object langv2 extends InstanceOfOpsImplicits { type Trav1[T] = T - implicit val trav1Ops = Trav1Ops - + trait TypeMultiplexer { + type CCOneToOne[_] + type CCOneToOption[_] + type CCOneToBoolean[_] + type CCOneToMany[_] + } - private val it2Ops = new IterableOps() - implicit def toIt2Ops[Col[A] <: Iterable[A]] = it2Ops.asInstanceOf[IterableOps[Col]] + object SingleTypes extends TypeMultiplexer { + override type CCOneToOne[T] = T + override type CCOneToOption[T] = Option[T] + override type CCOneToBoolean[T] = Option[T] + override type CCOneToMany[T] = Seq[T] + } - private val iterableOnceOpsOps = new IterableOnceOpsOps() - //implicit def toIterOnceOps[CC[_], C]: TravOps[({type X[+B] = IterableOnceOps[B, CC, C]})#X] = { - // iterableOnceOpsOps.asInstanceOf[TravOps[({type X[+B] = IterableOnceOps[B, CC, C]})#X]] - //} + object OptionTypes extends TypeMultiplexer { + override type CCOneToOne[T] = Option[T] + override type CCOneToOption[T] = Option[T] + override type CCOneToBoolean[T] = Option[T] + override type CCOneToMany[T] = Seq[T] + } - // This works if u provide CC and C eplicitly as [Iterable, Iterable[Any]] - implicit def toIterOnceOps[CC[_], C]: IterableOnceOpsOps[CC, C] = { - iterableOnceOpsOps.asInstanceOf[IterableOnceOpsOps[CC, C]] + class IterableTypes[CC[_], C] extends TypeMultiplexer { + override type CCOneToOne[T] = CC[T] + override type CCOneToOption[T] = CC[T] + override type CCOneToBoolean[T] = C + override type CCOneToMany[T] = CC[T] } + + implicit val trav1Ops = Trav1Ops + + + private val it2Ops = new IterableOnceOpsOps() + implicit def toIt2Ops[CC[_], C] = it2Ops.asInstanceOf[IterableOnceOpsOps[CC, C]] + // Iterable[A] = IterableOnceOps[A, Iterable, Iterable[A]] // io.shiftleft.semanticcpg.langv2.IterableOnceOpsOps[[+A]Iterable[A],Iterable[Any]] @@ -28,8 +47,15 @@ package object langv2 extends InstanceOfOpsImplicits { // implicit val iterOps = new IterableOnceOpsOps(Iterable.empty[Any]) - implicit def toAnyTraversal[I, IT[_]](trav: IT[I]): AnyTraversal[I, IT] = { - new AnyTraversal(trav) + implicit def toAnyTraversalNew1[I](trav: Option[I]) = { + new AnyTraversal[I, Option, OptionTypes.type](trav) + } + implicit def toAnyTraversalNew2[I, IT[_], TM <: TypeMultiplexer](trav: IT[I]) = { + new AnyTraversal[I, IT, TM](trav) + } + implicit def toAnyTraversalNew3[I, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { + //new AstTraversalNew[I, ({type X[B] = IterableOnceOps[B, CC, C]})#X, CC, CC, ({type X[B] = C})#X, CC](trav) + new AnyTraversal[I, ({type X[B] = IterableOnceOps[B, CC, C]})#X, IterableTypes[CC, C]](trav) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala index 8820f76b8..91aa744eb 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala @@ -12,13 +12,13 @@ import scala.jdk.CollectionConverters._ * A method, function, or procedure * */ @help.Traversal(elementType = classOf[nodes.Method]) -class MethodTraversal[I <: nodes.Method, IT[_]](val trav: IT[I]) extends AnyVal { +class MethodTraversal[I <: nodes.Method, IT[_], TM <: TypeMultiplexer](val trav: IT[I]) extends AnyVal { /** * Traverse to parameters of the method * */ @Doc("All parameters") - def parameter(implicit ops1: TravOps[IT]) = { + def parameter(implicit ops1: TravOps[IT, TM]) = { ops1.oneToMany(trav)(_._astOut.asScala.collect { case par: nodes.MethodParameterIn => par }) } @@ -26,7 +26,7 @@ class MethodTraversal[I <: nodes.Method, IT[_]](val trav: IT[I]) extends AnyVal * Traverse to formal return parameter * */ @Doc("All formal return parameters") - def methodReturn(implicit ops1: TravOps[IT]) = { + def methodReturn(implicit ops1: TravOps[IT, TM]) = { ops1.oneToOne(trav)(_._astOut.asScala.collectFirst { case ret: nodes.MethodReturn => ret }.get) } From 7b8e431148db74bc2ca68a682f2b591d9c7b1a48 Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Thu, 18 Nov 2021 21:31:19 +0100 Subject: [PATCH 10/27] Move IT[_] into TM --- .../src/main/scala/some/SomeDomain.scala | 12 +++--- .../shiftleft/semanticcpg/language/New.scala | 18 ++++----- .../generalizations/AstNodeTraversal.scala | 8 ++-- .../types/structure/LocalTraversal.scala | 4 +- .../semanticcpg/langv2/AnyTraversal.scala | 12 +++--- .../semanticcpg/langv2/TravOps.scala | 40 +++++++++---------- .../semanticcpg/langv2/package.scala | 12 ++++-- .../types/structure/MethodTraversal.scala | 6 +-- 8 files changed, 58 insertions(+), 54 deletions(-) diff --git a/performance/src/main/scala/some/SomeDomain.scala b/performance/src/main/scala/some/SomeDomain.scala index 33afc468d..5880ca51b 100644 --- a/performance/src/main/scala/some/SomeDomain.scala +++ b/performance/src/main/scala/some/SomeDomain.scala @@ -6,25 +6,25 @@ import scala.collection.IterableOnceOps object SomeDomain { implicit def toSynth1(p: D1) = { - new SynthExt[Trav1, SingleTypes.type](p: Trav1[D1]) + new SynthExt[SingleTypes.type](p: Trav1[D1]) } implicit def toSynth2(trav: Option[D1]) = { - new SynthExt[Option, OptionTypes.type](trav) + new SynthExt[OptionTypes.type](trav) } implicit def toSynth3[CC[_], C](trav: IterableOnceOps[D1, CC, C]) = { //new AstTraversalNew[I, ({type X[B] = IterableOnceOps[B, CC, C]})#X, CC, CC, ({type X[B] = C})#X, CC](trav) - new SynthExt[({type X[B] = IterableOnceOps[B, CC, C]})#X, IterableTypes[CC, C]](trav) + new SynthExt[IterableTypes[CC, C]](trav) } //implicit def toSynth[IT[_]](p: IT[D1]): SynthExt[IT] = { // new SynthExt(p) //} - class SynthExt[IT[_], TM <: TypeMultiplexer](val trav: IT[D1]) extends AnyVal { - def toD2(implicit ops: TravOps[IT, TM]) = { + class SynthExt[TM <: TypeMultiplexer](val trav: TM#IT[D1]) extends AnyVal { + def toD2(implicit ops: TravOps[TM]) = { ops.oneToOne(trav)(_.x) } - def toD2Multi(implicit ops: TravOps[IT, TM]) = { + def toD2Multi(implicit ops: TravOps[TM]) = { ops.oneToMany(trav)(x => Iterator.single(x)) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala index d0e66c97c..45f963545 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala @@ -11,43 +11,43 @@ import scala.collection.IterableOnceOps object New { implicit def toAstTraversalNew1[I <: nodes.AstNode](trav: I) = { - new AstTraversalNew[I, Trav1, SingleTypes.type](trav: Trav1[I]) + new AstTraversalNew[I, SingleTypes.type](trav: Trav1[I]) } implicit def toAstTraversalNew2[I <: nodes.AstNode](trav: Option[I]) = { - new AstTraversalNew[I, Option, OptionTypes.type](trav) + new AstTraversalNew[I, OptionTypes.type](trav) } //implicit def toAstTraversalNew2[I <: nodes.AstNode, IT[_]](trav: IT[I]): AstTraversalNew[I, IT] = { // new AstTraversalNew(trav) //} implicit def toAstTraversalNew3[I <: nodes.AstNode, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { //new AstTraversalNew[I, ({type X[B] = IterableOnceOps[B, CC, C]})#X, CC, CC, ({type X[B] = C})#X, CC](trav) - new AstTraversalNew[I, ({type X[B] = IterableOnceOps[B, CC, C]})#X, IterableTypes[CC, C]](trav) + new AstTraversalNew[I, IterableTypes[CC, C]](trav) } implicit def toMethodTraversalNew1[I <: nodes.Method](trav: I) = { - new MethodTraversal[I, Trav1, SingleTypes.type](trav: Trav1[I]) + new MethodTraversal[I, SingleTypes.type](trav: Trav1[I]) } implicit def toMethodTraversalNew2[I <: nodes.Method](trav: Option[I]) = { - new MethodTraversal[I, Option, OptionTypes.type](trav) + new MethodTraversal[I, OptionTypes.type](trav) } //implicit def toMethodTraversalNew2[I <: nodes.Method, IT[_]](trav: IT[I]): MethodTraversal[I, IT] = { // new MethodTraversal(trav) //} implicit def toMethodTraversalNew3[I <: nodes.Method, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - new MethodTraversal[I, ({type X[B] = IterableOnceOps[B, CC, C]})#X, IterableTypes[CC, C]](trav) + new MethodTraversal[I, IterableTypes[CC, C]](trav) } implicit def toLocalTraversalNew1[I <: nodes.Local](trav: I) = { - new LocalTraversalNew[I, Trav1, SingleTypes.type](trav: Trav1[I]) + new LocalTraversalNew[I, SingleTypes.type](trav: Trav1[I]) } implicit def toLocalTraversalNew2[I <: nodes.Local](trav: Option[I]) = { - new LocalTraversalNew[I, Option, OptionTypes.type](trav) + new LocalTraversalNew[I, OptionTypes.type](trav) } //implicit def toLocalTraversalNew2[I <: nodes.Local, IT[_]](trav: IT[I]): LocalTraversalNew[I, IT] = { // new LocalTraversalNew(trav) //} implicit def toLocalTraversalNew3[I <: nodes.Local, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - new LocalTraversalNew[I, ({type X[B] = IterableOnceOps[B, CC, C]})#X, IterableTypes[CC, C]](trav) + new LocalTraversalNew[I, IterableTypes[CC, C]](trav) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala index f3135ada7..4aed5f7e6 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala @@ -49,20 +49,20 @@ object AstNodeIsExpression { } } -class AstTraversalNew[I <: AstNode, IT[_], TM <: TypeMultiplexer](val trav: IT[I]) +class AstTraversalNew[I <: AstNode, TM <: TypeMultiplexer](val trav: TM#IT[I]) extends AnyVal { /** * Nodes of the AST rooted in this node, including the node itself. * */ @Doc("All nodes of the abstract syntax tree") - def ast(implicit ops1: TravOps[IT, TM]) = - trav.rFlatMap(_._astOut.asScala.asInstanceOf[Iterator[I]], _.emit).cast[AstNode] + def ast(implicit ops1: TravOps[TM]) = + trav.rFlatMap(_._astOut.asScala.asInstanceOf[Iterator[I]], _.emit) /** * Traverse only to AST nodes that are expressions * */ - def isExpression(implicit ops1: TravOps[IT, TM]) = + def isExpression(implicit ops1: TravOps[TM]) = trav.collectAll[Expression] } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala index 772b30b71..8f36dcefc 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala @@ -46,8 +46,8 @@ object LocalReferencingIdentifiers { } } -class LocalTraversalNew[I <: nodes.Local, IT[_], TM <: TypeMultiplexer](val trav: IT[I]) extends AnyVal { - def referencingIdentifiers(implicit ops1: TravOps[IT, TM]): TM#CCOneToMany[Identifier] = { +class LocalTraversalNew[I <: nodes.Local, TM <: TypeMultiplexer](val trav: TM#IT[I]) extends AnyVal { + def referencingIdentifiers(implicit ops1: TravOps[TM]): TM#CCOneToMany[Identifier] = { ops1.oneToMany(trav)(_._refIn.asScala.filter(_.label == NodeTypes.IDENTIFIER) .cast[Identifier]) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala index dc6c86deb..95051299e 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala @@ -3,37 +3,37 @@ package io.shiftleft.semanticcpg.langv2 import overflowdb.traversal.RepeatStepIterator -class AnyTraversal[I, IT[_], TM <: TypeMultiplexer](val trav: IT[I]) extends AnyVal { +class AnyTraversal[I, TM <: TypeMultiplexer](val trav: TM#IT[I]) extends AnyVal { def cast[A]: TM#CCOneToOne[A] = { trav.asInstanceOf[TM#CCOneToOne[A]] } // We dont use ClassTag and instead use our own IsInstanceOfOps for performance reasons. - def collectAll[T](implicit ops: TravOps[IT, TM], isInstanceOfOps: IsInstanceOfOps[T]): TM#CCOneToBoolean[T] = { + def collectAll[T](implicit ops: TravOps[TM], isInstanceOfOps: IsInstanceOfOps[T]): TM#CCOneToBoolean[T] = { ops.oneToBoolean(trav)(isInstanceOfOps).asInstanceOf[TM#CCOneToBoolean[T]] } def rMap[O <: I](f: I => O, g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[IT, TM]): TM#CCOneToMany[I] = { + implicit ops: TravOps[TM]): TM#CCOneToMany[I] = { val behaviour = g(new RepeatBehaviourBuilder()).build ops.oneToMany(trav)(i => new RepeatStepIterator(i, f.andThen(Iterator.single), behaviour)) } def rFlatMap[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[IT, TM]): TM#CCOneToMany[I] = { + implicit ops: TravOps[TM]): TM#CCOneToMany[I] = { val behaviour = g(new RepeatBehaviourBuilder()).build ops.oneToMany(trav)(i => new RepeatStepIterator(i, f, behaviour)) } @deprecated("Use rFlatMap instead") def repeat[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[IT, TM]): TM#CCOneToMany[I] = { + implicit ops: TravOps[TM]): TM#CCOneToMany[I] = { rFlatMap(f, g) } @deprecated("Calls to this function can be omitted") - def l: IT[I] = { + def l: TM#IT[I] = { trav } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala index a96f9e4ba..bc64f10eb 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala @@ -10,25 +10,25 @@ import scala.collection.IterableOnceOps * @tparam FlatMapTraversal Flat map operation output traversal type. * Abbreviated in other generics as FT. */ -trait TravOps[InputTraversal[_], TM <: TypeMultiplexer] { +trait TravOps[TM <: TypeMultiplexer] { type tm = TM - def oneToOne[I, O](trav: InputTraversal[I])(f: I => O): TM#CCOneToOne[O] - def oneToOption[I, O](trav: InputTraversal[I])(f: I => Option[O]): TM#CCOneToOption[O] - def oneToBoolean[I](trav: InputTraversal[I])(f: I => Boolean): TM#CCOneToBoolean[I] - def oneToMany[I, O](trav: InputTraversal[I])(f: I => Iterator[O]): TM#CCOneToMany[O] + def oneToOne[I, O](trav: TM#IT[I])(f: I => O): TM#CCOneToOne[O] + def oneToOption[I, O](trav: TM#IT[I])(f: I => Option[O]): TM#CCOneToOption[O] + def oneToBoolean[I](trav: TM#IT[I])(f: I => Boolean): TM#CCOneToBoolean[I] + def oneToMany[I, O](trav: TM#IT[I])(f: I => Iterator[O]): TM#CCOneToMany[O] } -object Trav1Ops extends TravOps[Trav1, SingleTypes.type] { +object Trav1Ops extends TravOps[SingleTypes.type] { - override def oneToOne[I, O](trav: Trav1[I])(f: I => O): this.type#tm#CCOneToOne[O] = { + override def oneToOne[I, O](trav: this.type#tm#IT[I])(f: I => O): this.type#tm#CCOneToOne[O] = { f(trav) } - override def oneToOption[I, O](trav: Trav1[I])(f: I => Option[O]): this.type#tm#CCOneToOption[O] = { + override def oneToOption[I, O](trav: this.type#tm#IT[I])(f: I => Option[O]): this.type#tm#CCOneToOption[O] = { f(trav) } - override def oneToBoolean[I](trav: Trav1[I])(f: I => Boolean): this.type#tm#CCOneToBoolean[I] = { + override def oneToBoolean[I](trav: this.type#tm#IT[I])(f: I => Boolean): this.type#tm#CCOneToBoolean[I] = { if (f(trav)) { Some(trav) } else { @@ -36,26 +36,26 @@ object Trav1Ops extends TravOps[Trav1, SingleTypes.type] { } } - override def oneToMany[I, O](trav: Trav1[I])(f: I => Iterator[O]): this.type#tm#CCOneToMany[O] = { + override def oneToMany[I, O](trav: this.type#tm#IT[I])(f: I => Iterator[O]): this.type#tm#CCOneToMany[O] = { Seq.from(f(trav)) } } -object OptionOps extends TravOps[Option, OptionTypes.type ] { +object OptionOps extends TravOps[OptionTypes.type ] { - override def oneToOne[I, O](trav: Option[I])(f: I => O): this.type#tm#CCOneToOne[O] = { + override def oneToOne[I, O](trav: this.type#tm#IT[I])(f: I => O): this.type#tm#CCOneToOne[O] = { trav.map(f) } - override def oneToOption[I, O](trav: Option[I])(f: I => Option[O]): this.type#tm#CCOneToOption[O] = { + override def oneToOption[I, O](trav: this.type#tm#IT[I])(f: I => Option[O]): this.type#tm#CCOneToOption[O] = { trav.flatMap(f) } - override def oneToBoolean[I](trav: Option[I])(f: I => Boolean): this.type#tm#CCOneToBoolean[I] = { + override def oneToBoolean[I](trav: this.type#tm#IT[I])(f: I => Boolean): this.type#tm#CCOneToBoolean[I] = { trav.filter(f) } - override def oneToMany[I, O](trav: Option[I])(f: I => Iterator[O]): this.type#tm#CCOneToMany[O] = { + override def oneToMany[I, O](trav: this.type#tm#IT[I])(f: I => Iterator[O]): this.type#tm#CCOneToMany[O] = { trav match { case Some(t) => Seq.from(f(t)) @@ -89,21 +89,21 @@ object OptionOps extends TravOps[Option, OptionTypes.type ] { //} //class IterableOnceOpsOps[CC[_], C](unused: IterableOnceOps[_, CC, C]) extends TravOps[({type X[B] = IterableOnceOps[B, CC, C]})#X] { -class IterableOnceOpsOps[CC[_], C] extends TravOps[({type X[B] = IterableOnceOps[B, CC, C]})#X, IterableTypes[CC, C] ] { +class IterableOnceOpsOps[CC[_], C] extends TravOps[IterableTypes[CC, C] ] { - override def oneToOne[I, O](trav: IterableOnceOps[I, CC, C])(f: I => O): this.type#tm#CCOneToOne[O] = { + override def oneToOne[I, O](trav: this.type#tm#IT[I])(f: I => O): this.type#tm#CCOneToOne[O] = { trav.map(f) } - override def oneToOption[I, O](trav: IterableOnceOps[I, CC, C])(f: I => Option[O]): this.type#tm#CCOneToOption[O] = { + override def oneToOption[I, O](trav: this.type#tm#IT[I])(f: I => Option[O]): this.type#tm#CCOneToOption[O] = { trav.flatMap(f) } - override def oneToBoolean[I](trav: IterableOnceOps[I, CC, C])(f: I => Boolean): this.type#tm#CCOneToBoolean[I] = { + override def oneToBoolean[I](trav: this.type#tm#IT[I])(f: I => Boolean): this.type#tm#CCOneToBoolean[I] = { trav.filter(f) } - override def oneToMany[I, O](trav: IterableOnceOps[I, CC, C])(f: I => Iterator[O]): this.type#tm#CCOneToMany[O] = { + override def oneToMany[I, O](trav: this.type#tm#IT[I])(f: I => Iterator[O]): this.type#tm#CCOneToMany[O] = { trav.flatMap(f) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala index 5d2bdfe35..91c6bc9a0 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala @@ -6,6 +6,7 @@ package object langv2 extends InstanceOfOpsImplicits { type Trav1[T] = T trait TypeMultiplexer { + type IT[_] type CCOneToOne[_] type CCOneToOption[_] type CCOneToBoolean[_] @@ -13,6 +14,7 @@ package object langv2 extends InstanceOfOpsImplicits { } object SingleTypes extends TypeMultiplexer { + override type IT[T] = T override type CCOneToOne[T] = T override type CCOneToOption[T] = Option[T] override type CCOneToBoolean[T] = Option[T] @@ -20,6 +22,7 @@ package object langv2 extends InstanceOfOpsImplicits { } object OptionTypes extends TypeMultiplexer { + override type IT[T] = Option[T] override type CCOneToOne[T] = Option[T] override type CCOneToOption[T] = Option[T] override type CCOneToBoolean[T] = Option[T] @@ -27,6 +30,7 @@ package object langv2 extends InstanceOfOpsImplicits { } class IterableTypes[CC[_], C] extends TypeMultiplexer { + override type IT[T] = IterableOnceOps[T, CC, C] override type CCOneToOne[T] = CC[T] override type CCOneToOption[T] = CC[T] override type CCOneToBoolean[T] = C @@ -48,14 +52,14 @@ package object langv2 extends InstanceOfOpsImplicits { // implicit val iterOps = new IterableOnceOpsOps(Iterable.empty[Any]) implicit def toAnyTraversalNew1[I](trav: Option[I]) = { - new AnyTraversal[I, Option, OptionTypes.type](trav) + new AnyTraversal[I, OptionTypes.type](trav) } - implicit def toAnyTraversalNew2[I, IT[_], TM <: TypeMultiplexer](trav: IT[I]) = { - new AnyTraversal[I, IT, TM](trav) + implicit def toAnyTraversalNew2[I, TM <: TypeMultiplexer](trav: TM#IT[I]) = { + new AnyTraversal[I, TM](trav) } implicit def toAnyTraversalNew3[I, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { //new AstTraversalNew[I, ({type X[B] = IterableOnceOps[B, CC, C]})#X, CC, CC, ({type X[B] = C})#X, CC](trav) - new AnyTraversal[I, ({type X[B] = IterableOnceOps[B, CC, C]})#X, IterableTypes[CC, C]](trav) + new AnyTraversal[I, IterableTypes[CC, C]](trav) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala index 91aa744eb..adc8fc92b 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala @@ -12,13 +12,13 @@ import scala.jdk.CollectionConverters._ * A method, function, or procedure * */ @help.Traversal(elementType = classOf[nodes.Method]) -class MethodTraversal[I <: nodes.Method, IT[_], TM <: TypeMultiplexer](val trav: IT[I]) extends AnyVal { +class MethodTraversal[I <: nodes.Method, TM <: TypeMultiplexer](val trav: TM#IT[I]) extends AnyVal { /** * Traverse to parameters of the method * */ @Doc("All parameters") - def parameter(implicit ops1: TravOps[IT, TM]) = { + def parameter(implicit ops1: TravOps[TM]) = { ops1.oneToMany(trav)(_._astOut.asScala.collect { case par: nodes.MethodParameterIn => par }) } @@ -26,7 +26,7 @@ class MethodTraversal[I <: nodes.Method, IT[_], TM <: TypeMultiplexer](val trav: * Traverse to formal return parameter * */ @Doc("All formal return parameters") - def methodReturn(implicit ops1: TravOps[IT, TM]) = { + def methodReturn(implicit ops1: TravOps[TM]) = { ops1.oneToOne(trav)(_._astOut.asScala.collectFirst { case ret: nodes.MethodReturn => ret }.get) } From f29283b846bd75870edf1a7a67758b3cfb16f74b Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Fri, 19 Nov 2021 12:59:01 +0100 Subject: [PATCH 11/27] Add TypeBound. --- performance/src/main/scala/MyTests.scala | 9 +++++-- .../src/main/scala/some/SomeDomain.scala | 4 +-- .../shiftleft/semanticcpg/language/New.scala | 25 +++++------------- .../semanticcpg/langv2/TravOps.scala | 26 +------------------ .../semanticcpg/langv2/package.scala | 16 ++++-------- 5 files changed, 22 insertions(+), 58 deletions(-) diff --git a/performance/src/main/scala/MyTests.scala b/performance/src/main/scala/MyTests.scala index 76ebe31b4..a03c658a0 100644 --- a/performance/src/main/scala/MyTests.scala +++ b/performance/src/main/scala/MyTests.scala @@ -11,7 +11,7 @@ import org.openjdk.jmh.annotations._ import MyTests._ import io.shiftleft.semanticcpg.language.types.structure.LocalReferencingIdentifiers -import scala.collection.mutable +import scala.collection.{View, mutable} import scala.collection.mutable.{ArrayBuffer, ListBuffer} import org.openjdk.jmh.infra.Blackhole import overflowdb.traversal.Traversal @@ -132,7 +132,6 @@ class MyTestNew { def syntheticIterableNew(state: MyState) = { //toSynth(state.d1:: Nil).toD2Multi //val y: TravOps[Iterable, IterableTypes[Iterable, Iterable[Any]]] = toIt2Ops - val c = toSynth3(Array(state.d1).view.slice(1,2)).toD2 val x: List[D2] = List(state.d1).toD2(toIt2Ops) x } @@ -141,6 +140,12 @@ class MyTestNew { def syntheticIterableBase(state: MyState) = { Iterable.single(state.d1).map(_.x) } + + def compileTest(state: MyState) = { + val c = toSynth3(Array(state.d1).view.slice(1,2)).toD2 + val d: View[D2] = c + d.rFlatMap(x => Iterator.single(x)).r + } } class MyTestsOld { diff --git a/performance/src/main/scala/some/SomeDomain.scala b/performance/src/main/scala/some/SomeDomain.scala index 5880ca51b..24ddca2d0 100644 --- a/performance/src/main/scala/some/SomeDomain.scala +++ b/performance/src/main/scala/some/SomeDomain.scala @@ -8,8 +8,8 @@ object SomeDomain { implicit def toSynth1(p: D1) = { new SynthExt[SingleTypes.type](p: Trav1[D1]) } - implicit def toSynth2(trav: Option[D1]) = { - new SynthExt[OptionTypes.type](trav) + implicit def toSynth2[IT[_]](trav: TypesClassFor[IT]#IT[D1]) = { + new SynthExt[TypesClassFor[IT]](trav) } implicit def toSynth3[CC[_], C](trav: IterableOnceOps[D1, CC, C]) = { //new AstTraversalNew[I, ({type X[B] = IterableOnceOps[B, CC, C]})#X, CC, CC, ({type X[B] = C})#X, CC](trav) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala index 45f963545..f43e1d70f 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala @@ -3,7 +3,7 @@ package io.shiftleft.semanticcpg.language import io.shiftleft.codepropertygraph.generated.nodes import io.shiftleft.semanticcpg.language.types.expressions.generalizations.AstTraversalNew import io.shiftleft.semanticcpg.language.types.structure.LocalTraversalNew -import io.shiftleft.semanticcpg.langv2.{IterableTypes, OptionTypes, SingleTypes, Trav1} +import io.shiftleft.semanticcpg.langv2.{IterableTypes, OptionTypes, SingleTypes, Trav1, TypeBound, TypesClassFor} import io.shiftleft.semanticcpg.langv2.types.structure.MethodTraversal import scala.collection.IterableOnceOps @@ -13,26 +13,19 @@ object New { implicit def toAstTraversalNew1[I <: nodes.AstNode](trav: I) = { new AstTraversalNew[I, SingleTypes.type](trav: Trav1[I]) } - implicit def toAstTraversalNew2[I <: nodes.AstNode](trav: Option[I]) = { - new AstTraversalNew[I, OptionTypes.type](trav) + implicit def toAstTraversalNew2[I <: nodes.AstNode, IT[_] <: TypeBound[_]](trav: TypesClassFor[IT]#IT[I]) = { + new AstTraversalNew[I, TypesClassFor[IT]](trav) } - //implicit def toAstTraversalNew2[I <: nodes.AstNode, IT[_]](trav: IT[I]): AstTraversalNew[I, IT] = { - // new AstTraversalNew(trav) - //} implicit def toAstTraversalNew3[I <: nodes.AstNode, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - //new AstTraversalNew[I, ({type X[B] = IterableOnceOps[B, CC, C]})#X, CC, CC, ({type X[B] = C})#X, CC](trav) new AstTraversalNew[I, IterableTypes[CC, C]](trav) } implicit def toMethodTraversalNew1[I <: nodes.Method](trav: I) = { new MethodTraversal[I, SingleTypes.type](trav: Trav1[I]) } - implicit def toMethodTraversalNew2[I <: nodes.Method](trav: Option[I]) = { - new MethodTraversal[I, OptionTypes.type](trav) + implicit def toMethodTraversalNew2[I <: nodes.Method, IT[_] <: TypeBound[_]](trav: TypesClassFor[IT]#IT[I]) = { + new MethodTraversal[I, TypesClassFor[IT]](trav) } - //implicit def toMethodTraversalNew2[I <: nodes.Method, IT[_]](trav: IT[I]): MethodTraversal[I, IT] = { - // new MethodTraversal(trav) - //} implicit def toMethodTraversalNew3[I <: nodes.Method, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { new MethodTraversal[I, IterableTypes[CC, C]](trav) } @@ -40,13 +33,9 @@ object New { implicit def toLocalTraversalNew1[I <: nodes.Local](trav: I) = { new LocalTraversalNew[I, SingleTypes.type](trav: Trav1[I]) } - implicit def toLocalTraversalNew2[I <: nodes.Local](trav: Option[I]) = { - new LocalTraversalNew[I, OptionTypes.type](trav) + implicit def toLocalTraversalNew2[I <: nodes.Local, IT[_] <: TypeBound[_]](trav: TypesClassFor[IT]#IT[I]) = { + new LocalTraversalNew[I, TypesClassFor[IT]](trav) } - - //implicit def toLocalTraversalNew2[I <: nodes.Local, IT[_]](trav: IT[I]): LocalTraversalNew[I, IT] = { - // new LocalTraversalNew(trav) - //} implicit def toLocalTraversalNew3[I <: nodes.Local, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { new LocalTraversalNew[I, IterableTypes[CC, C]](trav) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala index bc64f10eb..3a979ffa5 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala @@ -41,7 +41,7 @@ object Trav1Ops extends TravOps[SingleTypes.type] { } } -object OptionOps extends TravOps[OptionTypes.type ] { +object OptionOps extends TravOps[OptionTypes.type] { override def oneToOne[I, O](trav: this.type#tm#IT[I])(f: I => O): this.type#tm#CCOneToOne[O] = { trav.map(f) @@ -65,30 +65,6 @@ object OptionOps extends TravOps[OptionTypes.type ] { } } -//class IterableOps[Col[A] <: Iterable[A] ] extends TravOps[Col] { -// override type CCOneToOne[T] = Col[T] -// override type CCOneToOption[T] = Col[T] -// override type CCOneToBoolean[T] = Col[T] -// override type CCOneToMany[T] = Col[T] -// -// override def oneToOne[I, O](trav: Col[I])(f: I => O): CCOneToOne[O] = { -// trav.map(f).asInstanceOf[Col[O]] -// } -// -// override def oneToOption[I, O](trav: Col[I])(f: I => Option[O]): CCOneToOption[O] = { -// trav.flatMap(f).asInstanceOf[Col[O]] -// } -// -// override def oneToBoolean[I](trav: Col[I])(f: I => Boolean): CCOneToBoolean[I] = { -// trav.filter(f).asInstanceOf[Col[I]] -// } -// -// override def oneToMany[I, O](trav: Col[I])(f: I => Iterator[O]): CCOneToMany[O] = { -// trav.flatMap(f).asInstanceOf[Col[O]] -// } -//} - -//class IterableOnceOpsOps[CC[_], C](unused: IterableOnceOps[_, CC, C]) extends TravOps[({type X[B] = IterableOnceOps[B, CC, C]})#X] { class IterableOnceOpsOps[CC[_], C] extends TravOps[IterableTypes[CC, C] ] { override def oneToOne[I, O](trav: this.type#tm#IT[I])(f: I => O): this.type#tm#CCOneToOne[O] = { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala index 91c6bc9a0..32eaaa26c 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala @@ -37,28 +37,22 @@ package object langv2 extends InstanceOfOpsImplicits { override type CCOneToMany[T] = CC[T] } + type TypeBound[T] = Option[T] + type TypesClassFor[IT[_]] = OptionTypes.type implicit val trav1Ops = Trav1Ops - - + implicit val optionOps = OptionOps private val it2Ops = new IterableOnceOpsOps() implicit def toIt2Ops[CC[_], C] = it2Ops.asInstanceOf[IterableOnceOpsOps[CC, C]] - // Iterable[A] = IterableOnceOps[A, Iterable, Iterable[A]] - // io.shiftleft.semanticcpg.langv2.IterableOnceOpsOps[[+A]Iterable[A],Iterable[Any]] - - - - // implicit val iterOps = new IterableOnceOpsOps(Iterable.empty[Any]) - implicit def toAnyTraversalNew1[I](trav: Option[I]) = { - new AnyTraversal[I, OptionTypes.type](trav) + implicit def toAnyTraversalNew1[I, IT[_] <: TypeBound[_]](trav: TypesClassFor[IT]#IT[I]) = { + new AnyTraversal[I, TypesClassFor[IT]](trav) } implicit def toAnyTraversalNew2[I, TM <: TypeMultiplexer](trav: TM#IT[I]) = { new AnyTraversal[I, TM](trav) } implicit def toAnyTraversalNew3[I, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - //new AstTraversalNew[I, ({type X[B] = IterableOnceOps[B, CC, C]})#X, CC, CC, ({type X[B] = C})#X, CC](trav) new AnyTraversal[I, IterableTypes[CC, C]](trav) } From 0c0e60353ee908724a39c2b4cc075efbcc25771b Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Fri, 19 Nov 2021 14:48:32 +0100 Subject: [PATCH 12/27] Add back map, filter, flatMap to AnyTraversal. --- performance/src/main/scala/MyTests.scala | 13 +++---------- performance/src/main/scala/some/SomeDomain.scala | 4 ++-- .../language/types/structure/LocalTraversal.scala | 2 +- .../shiftleft/semanticcpg/langv2/AnyTraversal.scala | 12 ++++++++++++ 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/performance/src/main/scala/MyTests.scala b/performance/src/main/scala/MyTests.scala index a03c658a0..bd82af072 100644 --- a/performance/src/main/scala/MyTests.scala +++ b/performance/src/main/scala/MyTests.scala @@ -42,19 +42,13 @@ class MyTestNew { import io.shiftleft.semanticcpg.langv2._ import io.shiftleft.semanticcpg.language.New._ @Benchmark - def newTrav(state: MyState) = { - val x = toExtClass(state.method).methodReturn2() - x - } - - @Benchmark - def newTrav2(state: MyState) = { + def travNew(state: MyState) = { val x = state.method.methodReturn x } @Benchmark - def direct2(state: MyState) = { + def travBase(state: MyState) = { val x = state.method._astOut.asScala.collectFirst { case x: MethodReturn => x }.get x } @@ -144,7 +138,6 @@ class MyTestNew { def compileTest(state: MyState) = { val c = toSynth3(Array(state.d1).view.slice(1,2)).toD2 val d: View[D2] = c - d.rFlatMap(x => Iterator.single(x)).r } } @@ -152,7 +145,7 @@ class MyTestsOld { import io.shiftleft.semanticcpg.language._ @Benchmark - def oldTrav(state: MyState) = { + def travOld(state: MyState) = { val x = toMethodMethods(state.method).methodReturn.l x } diff --git a/performance/src/main/scala/some/SomeDomain.scala b/performance/src/main/scala/some/SomeDomain.scala index 24ddca2d0..7a9c25cc6 100644 --- a/performance/src/main/scala/some/SomeDomain.scala +++ b/performance/src/main/scala/some/SomeDomain.scala @@ -22,10 +22,10 @@ object SomeDomain { class SynthExt[TM <: TypeMultiplexer](val trav: TM#IT[D1]) extends AnyVal { def toD2(implicit ops: TravOps[TM]) = { - ops.oneToOne(trav)(_.x) + trav.map(_.x) } def toD2Multi(implicit ops: TravOps[TM]) = { - ops.oneToMany(trav)(x => Iterator.single(x)) + trav.flatMap(Iterator.single) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala index 8f36dcefc..f024f204d 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala @@ -48,7 +48,7 @@ object LocalReferencingIdentifiers { class LocalTraversalNew[I <: nodes.Local, TM <: TypeMultiplexer](val trav: TM#IT[I]) extends AnyVal { def referencingIdentifiers(implicit ops1: TravOps[TM]): TM#CCOneToMany[Identifier] = { - ops1.oneToMany(trav)(_._refIn.asScala.filter(_.label == NodeTypes.IDENTIFIER) + trav.flatMap(_._refIn.asScala.filter(_.label == NodeTypes.IDENTIFIER) .cast[Identifier]) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala index 95051299e..f0808a604 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala @@ -9,6 +9,18 @@ class AnyTraversal[I, TM <: TypeMultiplexer](val trav: TM#IT[I]) extends AnyVal trav.asInstanceOf[TM#CCOneToOne[A]] } + def map[O](f: I => O)(implicit ops: TravOps[TM]): TM#CCOneToOne[O] = { + ops.oneToOne(trav)(f) + } + + def filter(f: I => Boolean)(implicit ops: TravOps[TM]): TM#CCOneToBoolean[I] = { + ops.oneToBoolean(trav)(f) + } + + def flatMap[O](f: I => Iterator[O])(implicit ops: TravOps[TM]): TM#CCOneToMany[O] = { + ops.oneToMany(trav)(f) + } + // We dont use ClassTag and instead use our own IsInstanceOfOps for performance reasons. def collectAll[T](implicit ops: TravOps[TM], isInstanceOfOps: IsInstanceOfOps[T]): TM#CCOneToBoolean[T] = { ops.oneToBoolean(trav)(isInstanceOfOps).asInstanceOf[TM#CCOneToBoolean[T]] From 115bcf936b704b9be02a93bb481a9759bd617208 Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Fri, 19 Nov 2021 15:32:31 +0100 Subject: [PATCH 13/27] First round of renames. --- .../src/main/scala/some/SomeDomain.scala | 18 ++++---- .../shiftleft/semanticcpg/language/New.scala | 26 ++++++------ .../generalizations/AstNodeTraversal.scala | 6 +-- .../types/structure/LocalTraversal.scala | 4 +- .../semanticcpg/langv2/AnyTraversal.scala | 24 +++++------ .../semanticcpg/langv2/TravOps.scala | 42 +++++++++---------- .../semanticcpg/langv2/package.scala | 34 +++++++-------- .../types/structure/MethodTraversal.scala | 6 +-- 8 files changed, 78 insertions(+), 82 deletions(-) diff --git a/performance/src/main/scala/some/SomeDomain.scala b/performance/src/main/scala/some/SomeDomain.scala index 7a9c25cc6..68594fdf6 100644 --- a/performance/src/main/scala/some/SomeDomain.scala +++ b/performance/src/main/scala/some/SomeDomain.scala @@ -6,25 +6,21 @@ import scala.collection.IterableOnceOps object SomeDomain { implicit def toSynth1(p: D1) = { - new SynthExt[SingleTypes.type](p: Trav1[D1]) + new SynthExt[SingleTravTypes.type](p: Single[D1]) } - implicit def toSynth2[IT[_]](trav: TypesClassFor[IT]#IT[D1]) = { - new SynthExt[TypesClassFor[IT]](trav) + implicit def toSynth2[T[_]](trav: TravTypesFor[T]#Collection[D1]) = { + new SynthExt[TravTypesFor[T]](trav) } implicit def toSynth3[CC[_], C](trav: IterableOnceOps[D1, CC, C]) = { //new AstTraversalNew[I, ({type X[B] = IterableOnceOps[B, CC, C]})#X, CC, CC, ({type X[B] = C})#X, CC](trav) - new SynthExt[IterableTypes[CC, C]](trav) + new SynthExt[IterableOnceOpsTravTypes[CC, C]](trav) } - //implicit def toSynth[IT[_]](p: IT[D1]): SynthExt[IT] = { - // new SynthExt(p) - //} - - class SynthExt[TM <: TypeMultiplexer](val trav: TM#IT[D1]) extends AnyVal { - def toD2(implicit ops: TravOps[TM]) = { + class SynthExt[TT <: TravTypes](val trav: TT#Collection[D1]) extends AnyVal { + def toD2(implicit ops: TravOps[TT]) = { trav.map(_.x) } - def toD2Multi(implicit ops: TravOps[TM]) = { + def toD2Multi(implicit ops: TravOps[TT]) = { trav.flatMap(Iterator.single) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala index f43e1d70f..dfabb439c 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala @@ -3,7 +3,7 @@ package io.shiftleft.semanticcpg.language import io.shiftleft.codepropertygraph.generated.nodes import io.shiftleft.semanticcpg.language.types.expressions.generalizations.AstTraversalNew import io.shiftleft.semanticcpg.language.types.structure.LocalTraversalNew -import io.shiftleft.semanticcpg.langv2.{IterableTypes, OptionTypes, SingleTypes, Trav1, TypeBound, TypesClassFor} +import io.shiftleft.semanticcpg.langv2.{IterableOnceOpsTravTypes, OptionTravTypes, SingleTravTypes, Single, SupportedTypes, TravTypesFor} import io.shiftleft.semanticcpg.langv2.types.structure.MethodTraversal import scala.collection.IterableOnceOps @@ -11,32 +11,32 @@ import scala.collection.IterableOnceOps object New { implicit def toAstTraversalNew1[I <: nodes.AstNode](trav: I) = { - new AstTraversalNew[I, SingleTypes.type](trav: Trav1[I]) + new AstTraversalNew[I, SingleTravTypes.type](trav: Single[I]) } - implicit def toAstTraversalNew2[I <: nodes.AstNode, IT[_] <: TypeBound[_]](trav: TypesClassFor[IT]#IT[I]) = { - new AstTraversalNew[I, TypesClassFor[IT]](trav) + implicit def toAstTraversalNew2[I <: nodes.AstNode, T[_] <: SupportedTypes[_]](trav: TravTypesFor[T]#Collection[I]) = { + new AstTraversalNew[I, TravTypesFor[T]](trav) } implicit def toAstTraversalNew3[I <: nodes.AstNode, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - new AstTraversalNew[I, IterableTypes[CC, C]](trav) + new AstTraversalNew[I, IterableOnceOpsTravTypes[CC, C]](trav) } implicit def toMethodTraversalNew1[I <: nodes.Method](trav: I) = { - new MethodTraversal[I, SingleTypes.type](trav: Trav1[I]) + new MethodTraversal[I, SingleTravTypes.type](trav: Single[I]) } - implicit def toMethodTraversalNew2[I <: nodes.Method, IT[_] <: TypeBound[_]](trav: TypesClassFor[IT]#IT[I]) = { - new MethodTraversal[I, TypesClassFor[IT]](trav) + implicit def toMethodTraversalNew2[I <: nodes.Method, T[_] <: SupportedTypes[_]](trav: TravTypesFor[T]#Collection[I]) = { + new MethodTraversal[I, TravTypesFor[T]](trav) } implicit def toMethodTraversalNew3[I <: nodes.Method, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - new MethodTraversal[I, IterableTypes[CC, C]](trav) + new MethodTraversal[I, IterableOnceOpsTravTypes[CC, C]](trav) } implicit def toLocalTraversalNew1[I <: nodes.Local](trav: I) = { - new LocalTraversalNew[I, SingleTypes.type](trav: Trav1[I]) + new LocalTraversalNew[I, SingleTravTypes.type](trav: Single[I]) } - implicit def toLocalTraversalNew2[I <: nodes.Local, IT[_] <: TypeBound[_]](trav: TypesClassFor[IT]#IT[I]) = { - new LocalTraversalNew[I, TypesClassFor[IT]](trav) + implicit def toLocalTraversalNew2[I <: nodes.Local, T[_] <: SupportedTypes[_]](trav: TravTypesFor[T]#Collection[I]) = { + new LocalTraversalNew[I, TravTypesFor[T]](trav) } implicit def toLocalTraversalNew3[I <: nodes.Local, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - new LocalTraversalNew[I, IterableTypes[CC, C]](trav) + new LocalTraversalNew[I, IterableOnceOpsTravTypes[CC, C]](trav) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala index 4aed5f7e6..5a6c0e815 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala @@ -49,20 +49,20 @@ object AstNodeIsExpression { } } -class AstTraversalNew[I <: AstNode, TM <: TypeMultiplexer](val trav: TM#IT[I]) +class AstTraversalNew[I <: AstNode, TT <: TravTypes](val trav: TT#Collection[I]) extends AnyVal { /** * Nodes of the AST rooted in this node, including the node itself. * */ @Doc("All nodes of the abstract syntax tree") - def ast(implicit ops1: TravOps[TM]) = + def ast(implicit ops1: TravOps[TT]) = trav.rFlatMap(_._astOut.asScala.asInstanceOf[Iterator[I]], _.emit) /** * Traverse only to AST nodes that are expressions * */ - def isExpression(implicit ops1: TravOps[TM]) = + def isExpression(implicit ops1: TravOps[TT]) = trav.collectAll[Expression] } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala index f024f204d..80c0abb29 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala @@ -46,8 +46,8 @@ object LocalReferencingIdentifiers { } } -class LocalTraversalNew[I <: nodes.Local, TM <: TypeMultiplexer](val trav: TM#IT[I]) extends AnyVal { - def referencingIdentifiers(implicit ops1: TravOps[TM]): TM#CCOneToMany[Identifier] = { +class LocalTraversalNew[I <: nodes.Local, TT <: TravTypes](val trav: TT#Collection[I]) extends AnyVal { + def referencingIdentifiers(implicit ops1: TravOps[TT]): TT#CCOneToMany[Identifier] = { trav.flatMap(_._refIn.asScala.filter(_.label == NodeTypes.IDENTIFIER) .cast[Identifier]) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala index f0808a604..aa5002301 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala @@ -3,49 +3,49 @@ package io.shiftleft.semanticcpg.langv2 import overflowdb.traversal.RepeatStepIterator -class AnyTraversal[I, TM <: TypeMultiplexer](val trav: TM#IT[I]) extends AnyVal { +class AnyTraversal[I, TT <: TravTypes](val trav: TT#Collection[I]) extends AnyVal { - def cast[A]: TM#CCOneToOne[A] = { - trav.asInstanceOf[TM#CCOneToOne[A]] + def cast[A]: TT#CCOneToOne[A] = { + trav.asInstanceOf[TT#CCOneToOne[A]] } - def map[O](f: I => O)(implicit ops: TravOps[TM]): TM#CCOneToOne[O] = { + def map[O](f: I => O)(implicit ops: TravOps[TT]): TT#CCOneToOne[O] = { ops.oneToOne(trav)(f) } - def filter(f: I => Boolean)(implicit ops: TravOps[TM]): TM#CCOneToBoolean[I] = { + def filter(f: I => Boolean)(implicit ops: TravOps[TT]): TT#CCOneToBoolean[I] = { ops.oneToBoolean(trav)(f) } - def flatMap[O](f: I => Iterator[O])(implicit ops: TravOps[TM]): TM#CCOneToMany[O] = { + def flatMap[O](f: I => Iterator[O])(implicit ops: TravOps[TT]): TT#CCOneToMany[O] = { ops.oneToMany(trav)(f) } // We dont use ClassTag and instead use our own IsInstanceOfOps for performance reasons. - def collectAll[T](implicit ops: TravOps[TM], isInstanceOfOps: IsInstanceOfOps[T]): TM#CCOneToBoolean[T] = { - ops.oneToBoolean(trav)(isInstanceOfOps).asInstanceOf[TM#CCOneToBoolean[T]] + def collectAll[T](implicit ops: TravOps[TT], isInstanceOfOps: IsInstanceOfOps[T]): TT#CCOneToBoolean[T] = { + ops.oneToBoolean(trav)(isInstanceOfOps).asInstanceOf[TT#CCOneToBoolean[T]] } def rMap[O <: I](f: I => O, g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[TM]): TM#CCOneToMany[I] = { + implicit ops: TravOps[TT]): TT#CCOneToMany[I] = { val behaviour = g(new RepeatBehaviourBuilder()).build ops.oneToMany(trav)(i => new RepeatStepIterator(i, f.andThen(Iterator.single), behaviour)) } def rFlatMap[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[TM]): TM#CCOneToMany[I] = { + implicit ops: TravOps[TT]): TT#CCOneToMany[I] = { val behaviour = g(new RepeatBehaviourBuilder()).build ops.oneToMany(trav)(i => new RepeatStepIterator(i, f, behaviour)) } @deprecated("Use rFlatMap instead") def repeat[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[TM]): TM#CCOneToMany[I] = { + implicit ops: TravOps[TT]): TT#CCOneToMany[I] = { rFlatMap(f, g) } @deprecated("Calls to this function can be omitted") - def l: TM#IT[I] = { + def l: TT#Collection[I] = { trav } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala index 3a979ffa5..43c5b6f07 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala @@ -10,25 +10,25 @@ import scala.collection.IterableOnceOps * @tparam FlatMapTraversal Flat map operation output traversal type. * Abbreviated in other generics as FT. */ -trait TravOps[TM <: TypeMultiplexer] { - type tm = TM - def oneToOne[I, O](trav: TM#IT[I])(f: I => O): TM#CCOneToOne[O] - def oneToOption[I, O](trav: TM#IT[I])(f: I => Option[O]): TM#CCOneToOption[O] - def oneToBoolean[I](trav: TM#IT[I])(f: I => Boolean): TM#CCOneToBoolean[I] - def oneToMany[I, O](trav: TM#IT[I])(f: I => Iterator[O]): TM#CCOneToMany[O] +trait TravOps[_TT <: TravTypes] { + type TT = _TT + def oneToOne[I, O](trav: TT#Collection[I])(f: I => O): TT#CCOneToOne[O] + def oneToOption[I, O](trav: TT#Collection[I])(f: I => Option[O]): TT#CCOneToOption[O] + def oneToBoolean[I](trav: TT#Collection[I])(f: I => Boolean): TT#CCOneToBoolean[I] + def oneToMany[I, O](trav: TT#Collection[I])(f: I => Iterator[O]): TT#CCOneToMany[O] } -object Trav1Ops extends TravOps[SingleTypes.type] { +object SingleOps extends TravOps[SingleTravTypes.type] { - override def oneToOne[I, O](trav: this.type#tm#IT[I])(f: I => O): this.type#tm#CCOneToOne[O] = { + override def oneToOne[I, O](trav: this.type#TT#Collection[I])(f: I => O): this.type#TT#CCOneToOne[O] = { f(trav) } - override def oneToOption[I, O](trav: this.type#tm#IT[I])(f: I => Option[O]): this.type#tm#CCOneToOption[O] = { + override def oneToOption[I, O](trav: this.type#TT#Collection[I])(f: I => Option[O]): this.type#TT#CCOneToOption[O] = { f(trav) } - override def oneToBoolean[I](trav: this.type#tm#IT[I])(f: I => Boolean): this.type#tm#CCOneToBoolean[I] = { + override def oneToBoolean[I](trav: this.type#TT#Collection[I])(f: I => Boolean): this.type#TT#CCOneToBoolean[I] = { if (f(trav)) { Some(trav) } else { @@ -36,26 +36,26 @@ object Trav1Ops extends TravOps[SingleTypes.type] { } } - override def oneToMany[I, O](trav: this.type#tm#IT[I])(f: I => Iterator[O]): this.type#tm#CCOneToMany[O] = { + override def oneToMany[I, O](trav: this.type#TT#Collection[I])(f: I => Iterator[O]): this.type#TT#CCOneToMany[O] = { Seq.from(f(trav)) } } -object OptionOps extends TravOps[OptionTypes.type] { +object OptionOps extends TravOps[OptionTravTypes.type] { - override def oneToOne[I, O](trav: this.type#tm#IT[I])(f: I => O): this.type#tm#CCOneToOne[O] = { + override def oneToOne[I, O](trav: this.type#TT#Collection[I])(f: I => O): this.type#TT#CCOneToOne[O] = { trav.map(f) } - override def oneToOption[I, O](trav: this.type#tm#IT[I])(f: I => Option[O]): this.type#tm#CCOneToOption[O] = { + override def oneToOption[I, O](trav: this.type#TT#Collection[I])(f: I => Option[O]): this.type#TT#CCOneToOption[O] = { trav.flatMap(f) } - override def oneToBoolean[I](trav: this.type#tm#IT[I])(f: I => Boolean): this.type#tm#CCOneToBoolean[I] = { + override def oneToBoolean[I](trav: this.type#TT#Collection[I])(f: I => Boolean): this.type#TT#CCOneToBoolean[I] = { trav.filter(f) } - override def oneToMany[I, O](trav: this.type#tm#IT[I])(f: I => Iterator[O]): this.type#tm#CCOneToMany[O] = { + override def oneToMany[I, O](trav: this.type#TT#Collection[I])(f: I => Iterator[O]): this.type#TT#CCOneToMany[O] = { trav match { case Some(t) => Seq.from(f(t)) @@ -65,21 +65,21 @@ object OptionOps extends TravOps[OptionTypes.type] { } } -class IterableOnceOpsOps[CC[_], C] extends TravOps[IterableTypes[CC, C] ] { +class IterableOnceOpsOps[CC[_], C] extends TravOps[IterableOnceOpsTravTypes[CC, C] ] { - override def oneToOne[I, O](trav: this.type#tm#IT[I])(f: I => O): this.type#tm#CCOneToOne[O] = { + override def oneToOne[I, O](trav: this.type#TT#Collection[I])(f: I => O): this.type#TT#CCOneToOne[O] = { trav.map(f) } - override def oneToOption[I, O](trav: this.type#tm#IT[I])(f: I => Option[O]): this.type#tm#CCOneToOption[O] = { + override def oneToOption[I, O](trav: this.type#TT#Collection[I])(f: I => Option[O]): this.type#TT#CCOneToOption[O] = { trav.flatMap(f) } - override def oneToBoolean[I](trav: this.type#tm#IT[I])(f: I => Boolean): this.type#tm#CCOneToBoolean[I] = { + override def oneToBoolean[I](trav: this.type#TT#Collection[I])(f: I => Boolean): this.type#TT#CCOneToBoolean[I] = { trav.filter(f) } - override def oneToMany[I, O](trav: this.type#tm#IT[I])(f: I => Iterator[O]): this.type#tm#CCOneToMany[O] = { + override def oneToMany[I, O](trav: this.type#TT#Collection[I])(f: I => Iterator[O]): this.type#TT#CCOneToMany[O] = { trav.flatMap(f) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala index 32eaaa26c..db903d191 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala @@ -3,57 +3,57 @@ package io.shiftleft.semanticcpg import scala.collection.{IterableFactory, IterableOnceOps} package object langv2 extends InstanceOfOpsImplicits { - type Trav1[T] = T + type Single[T] = T - trait TypeMultiplexer { - type IT[_] + trait TravTypes { + type Collection[_] type CCOneToOne[_] type CCOneToOption[_] type CCOneToBoolean[_] type CCOneToMany[_] } - object SingleTypes extends TypeMultiplexer { - override type IT[T] = T + object SingleTravTypes extends TravTypes { + override type Collection[T] = T override type CCOneToOne[T] = T override type CCOneToOption[T] = Option[T] override type CCOneToBoolean[T] = Option[T] override type CCOneToMany[T] = Seq[T] } - object OptionTypes extends TypeMultiplexer { - override type IT[T] = Option[T] + object OptionTravTypes extends TravTypes { + override type Collection[T] = Option[T] override type CCOneToOne[T] = Option[T] override type CCOneToOption[T] = Option[T] override type CCOneToBoolean[T] = Option[T] override type CCOneToMany[T] = Seq[T] } - class IterableTypes[CC[_], C] extends TypeMultiplexer { - override type IT[T] = IterableOnceOps[T, CC, C] + class IterableOnceOpsTravTypes[CC[_], C] extends TravTypes { + override type Collection[T] = IterableOnceOps[T, CC, C] override type CCOneToOne[T] = CC[T] override type CCOneToOption[T] = CC[T] override type CCOneToBoolean[T] = C override type CCOneToMany[T] = CC[T] } - type TypeBound[T] = Option[T] - type TypesClassFor[IT[_]] = OptionTypes.type + type SupportedTypes[T] = Option[T] + type TravTypesFor[T[_]] = OptionTravTypes.type - implicit val trav1Ops = Trav1Ops + implicit val singleOps = SingleOps implicit val optionOps = OptionOps private val it2Ops = new IterableOnceOpsOps() implicit def toIt2Ops[CC[_], C] = it2Ops.asInstanceOf[IterableOnceOpsOps[CC, C]] - implicit def toAnyTraversalNew1[I, IT[_] <: TypeBound[_]](trav: TypesClassFor[IT]#IT[I]) = { - new AnyTraversal[I, TypesClassFor[IT]](trav) + implicit def toAnyTraversalNew1[I, T[_] <: SupportedTypes[_]](trav: TravTypesFor[T]#Collection[I]) = { + new AnyTraversal[I, TravTypesFor[T]](trav) } - implicit def toAnyTraversalNew2[I, TM <: TypeMultiplexer](trav: TM#IT[I]) = { - new AnyTraversal[I, TM](trav) + implicit def toAnyTraversalNew2[I, TT <: TravTypes](trav: TT#Collection[I]) = { + new AnyTraversal[I, TT](trav) } implicit def toAnyTraversalNew3[I, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - new AnyTraversal[I, IterableTypes[CC, C]](trav) + new AnyTraversal[I, IterableOnceOpsTravTypes[CC, C]](trav) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala index adc8fc92b..79dc4d0e5 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala @@ -12,13 +12,13 @@ import scala.jdk.CollectionConverters._ * A method, function, or procedure * */ @help.Traversal(elementType = classOf[nodes.Method]) -class MethodTraversal[I <: nodes.Method, TM <: TypeMultiplexer](val trav: TM#IT[I]) extends AnyVal { +class MethodTraversal[I <: nodes.Method, TT <: TravTypes](val trav: TT#Collection[I]) extends AnyVal { /** * Traverse to parameters of the method * */ @Doc("All parameters") - def parameter(implicit ops1: TravOps[TM]) = { + def parameter(implicit ops1: TravOps[TT]) = { ops1.oneToMany(trav)(_._astOut.asScala.collect { case par: nodes.MethodParameterIn => par }) } @@ -26,7 +26,7 @@ class MethodTraversal[I <: nodes.Method, TM <: TypeMultiplexer](val trav: TM#IT[ * Traverse to formal return parameter * */ @Doc("All formal return parameters") - def methodReturn(implicit ops1: TravOps[TM]) = { + def methodReturn(implicit ops1: TravOps[TT]) = { ops1.oneToOne(trav)(_._astOut.asScala.collectFirst { case ret: nodes.MethodReturn => ret }.get) } From a8772931d90906b62aeaf10121b836db250abf43 Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Fri, 19 Nov 2021 15:45:35 +0100 Subject: [PATCH 14/27] Moved implicits to classes they create and ... moved new AstNode and Local traversals into their own files. --- performance/src/main/scala/MyTests.scala | 4 +- .../shiftleft/semanticcpg/language/New.scala | 45 +++---------------- .../generalizations/AstNodeTraversal.scala | 22 +-------- .../types/structure/LocalTraversal.scala | 7 --- .../generalizations/AstNodeTraversal.scala | 41 +++++++++++++++++ .../types/structure/LocalTraversal.scala | 28 ++++++++++++ .../types/structure/MethodTraversal.scala | 12 +++++ 7 files changed, 91 insertions(+), 68 deletions(-) create mode 100644 semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala create mode 100644 semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala diff --git a/performance/src/main/scala/MyTests.scala b/performance/src/main/scala/MyTests.scala index bd82af072..36342e2f1 100644 --- a/performance/src/main/scala/MyTests.scala +++ b/performance/src/main/scala/MyTests.scala @@ -61,7 +61,7 @@ class MyTestNew { @Benchmark def refIdNew2(state: MyState) = { - val x = toLocalTraversalNew1(state.local).referencingIdentifiers + val x = toLocalTraversalSingle(state.local).referencingIdentifiers x } @@ -98,7 +98,7 @@ class MyTestNew { @Benchmark def astTestNewV2(state: MyState) = { - val x: Option[Expression] = toAstTraversalNew1(state.method).isExpression + val x: Option[Expression] = toAstNodeTraversalSingle(state.method).isExpression x } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala index dfabb439c..75fd0f6ba 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala @@ -1,42 +1,9 @@ package io.shiftleft.semanticcpg.language -import io.shiftleft.codepropertygraph.generated.nodes -import io.shiftleft.semanticcpg.language.types.expressions.generalizations.AstTraversalNew -import io.shiftleft.semanticcpg.language.types.structure.LocalTraversalNew -import io.shiftleft.semanticcpg.langv2.{IterableOnceOpsTravTypes, OptionTravTypes, SingleTravTypes, Single, SupportedTypes, TravTypesFor} -import io.shiftleft.semanticcpg.langv2.types.structure.MethodTraversal +import io.shiftleft.semanticcpg.langv2.types.expressions.generalizations.AstNodeTraversalImplicits +import io.shiftleft.semanticcpg.langv2.types.structure.{LocalTraversalImplicits, MethodTraversalImplicits} -import scala.collection.IterableOnceOps - -object New { - - implicit def toAstTraversalNew1[I <: nodes.AstNode](trav: I) = { - new AstTraversalNew[I, SingleTravTypes.type](trav: Single[I]) - } - implicit def toAstTraversalNew2[I <: nodes.AstNode, T[_] <: SupportedTypes[_]](trav: TravTypesFor[T]#Collection[I]) = { - new AstTraversalNew[I, TravTypesFor[T]](trav) - } - implicit def toAstTraversalNew3[I <: nodes.AstNode, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - new AstTraversalNew[I, IterableOnceOpsTravTypes[CC, C]](trav) - } - - implicit def toMethodTraversalNew1[I <: nodes.Method](trav: I) = { - new MethodTraversal[I, SingleTravTypes.type](trav: Single[I]) - } - implicit def toMethodTraversalNew2[I <: nodes.Method, T[_] <: SupportedTypes[_]](trav: TravTypesFor[T]#Collection[I]) = { - new MethodTraversal[I, TravTypesFor[T]](trav) - } - implicit def toMethodTraversalNew3[I <: nodes.Method, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - new MethodTraversal[I, IterableOnceOpsTravTypes[CC, C]](trav) - } - - implicit def toLocalTraversalNew1[I <: nodes.Local](trav: I) = { - new LocalTraversalNew[I, SingleTravTypes.type](trav: Single[I]) - } - implicit def toLocalTraversalNew2[I <: nodes.Local, T[_] <: SupportedTypes[_]](trav: TravTypesFor[T]#Collection[I]) = { - new LocalTraversalNew[I, TravTypesFor[T]](trav) - } - implicit def toLocalTraversalNew3[I <: nodes.Local, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - new LocalTraversalNew[I, IterableOnceOpsTravTypes[CC, C]](trav) - } -} +object New + extends MethodTraversalImplicits + with AstNodeTraversalImplicits + with LocalTraversalImplicits \ No newline at end of file diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala index 5a6c0e815..29cd47f20 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala @@ -1,14 +1,14 @@ package io.shiftleft.semanticcpg.language.types.expressions.generalizations import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} +import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, nodes} import io.shiftleft.semanticcpg.language._ import io.shiftleft.semanticcpg.langv2._ import io.shiftleft.semanticcpg.langv3.{Helper, Kernel1ToN, Kernel1ToO} import overflowdb.traversal.help.Doc import overflowdb.traversal.{Traversal, help} -import scala.collection.IterableOps +import scala.collection.{IterableOnceOps, IterableOps} import scala.jdk.CollectionConverters._ class AstNodeIsExpressionKernel[I <: AstNode] extends Kernel1ToO[I, I with Expression] { @@ -49,24 +49,6 @@ object AstNodeIsExpression { } } -class AstTraversalNew[I <: AstNode, TT <: TravTypes](val trav: TT#Collection[I]) - extends AnyVal { - - /** - * Nodes of the AST rooted in this node, including the node itself. - * */ - @Doc("All nodes of the abstract syntax tree") - def ast(implicit ops1: TravOps[TT]) = - trav.rFlatMap(_._astOut.asScala.asInstanceOf[Iterator[I]], _.emit) - - /** - * Traverse only to AST nodes that are expressions - * */ - def isExpression(implicit ops1: TravOps[TT]) = - trav.collectAll[Expression] - -} - @help.Traversal(elementType = classOf[AstNode]) class AstNodeTraversal[A <: AstNode](val traversal: Traversal[A]) extends AnyVal { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala index 80c0abb29..488c17d07 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala @@ -46,13 +46,6 @@ object LocalReferencingIdentifiers { } } -class LocalTraversalNew[I <: nodes.Local, TT <: TravTypes](val trav: TT#Collection[I]) extends AnyVal { - def referencingIdentifiers(implicit ops1: TravOps[TT]): TT#CCOneToMany[Identifier] = { - trav.flatMap(_._refIn.asScala.filter(_.label == NodeTypes.IDENTIFIER) - .cast[Identifier]) - } -} - /** * A local variable * */ diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala new file mode 100644 index 000000000..f518da80c --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala @@ -0,0 +1,41 @@ +package io.shiftleft.semanticcpg.langv2.types.expressions.generalizations + +import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, Expression} +import io.shiftleft.semanticcpg.langv2._ +import overflowdb.traversal.help.Doc + +import scala.collection.IterableOnceOps +import scala.jdk.CollectionConverters._ + + +trait AstNodeTraversalImplicits { + implicit def toAstNodeTraversalSingle[I <: AstNode](trav: I) = { + new AstNodeTraversal[I, SingleTravTypes.type](trav: Single[I]) + } + implicit def toAstNodeTraversalGeneric[I <: AstNode, T[_] <: SupportedTypes[_]](trav: TravTypesFor[T]#Collection[I]) = { + new AstNodeTraversal[I, TravTypesFor[T]](trav) + } + implicit def toAstNodeTraversalIterOnceOps[I <: AstNode, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { + new AstNodeTraversal[I, IterableOnceOpsTravTypes[CC, C]](trav) + } + +} + +class AstNodeTraversal[I <: AstNode, TT <: TravTypes](val trav: TT#Collection[I]) + extends AnyVal { + + /** + * Nodes of the AST rooted in this node, including the node itself. + * */ + @Doc("All nodes of the abstract syntax tree") + def ast(implicit ops1: TravOps[TT]) = + trav.rFlatMap(_._astOut.asScala.asInstanceOf[Iterator[I]], _.emit) + + /** + * Traverse only to AST nodes that are expressions + * */ + def isExpression(implicit ops1: TravOps[TT]) = + trav.collectAll[Expression] + +} + diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala new file mode 100644 index 000000000..3cc5b5df7 --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala @@ -0,0 +1,28 @@ +package io.shiftleft.semanticcpg.langv2.types.structure + +import io.shiftleft.codepropertygraph.generated.nodes.Identifier +import io.shiftleft.codepropertygraph.generated.{NodeTypes, nodes} +import io.shiftleft.semanticcpg.langv2._ + +import scala.collection.IterableOnceOps +import scala.jdk.CollectionConverters._ + +trait LocalTraversalImplicits { + + implicit def toLocalTraversalSingle[I <: nodes.Local](trav: I) = { + new LocalTraversal[I, SingleTravTypes.type](trav: Single[I]) + } + implicit def toLocalTraversalGeneric[I <: nodes.Local, T[_] <: SupportedTypes[_]](trav: TravTypesFor[T]#Collection[I]) = { + new LocalTraversal[I, TravTypesFor[T]](trav) + } + implicit def toLocalTraversalIterOnceOps[I <: nodes.Local, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { + new LocalTraversal[I, IterableOnceOpsTravTypes[CC, C]](trav) + } +} + +class LocalTraversal[I <: nodes.Local, TT <: TravTypes](val trav: TT#Collection[I]) extends AnyVal { + def referencingIdentifiers(implicit ops1: TravOps[TT]): TT#CCOneToMany[Identifier] = { + trav.flatMap(_._refIn.asScala.filter(_.label == NodeTypes.IDENTIFIER) + .cast[Identifier]) + } +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala index 79dc4d0e5..baf3c81ea 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala @@ -5,8 +5,20 @@ import io.shiftleft.semanticcpg.langv2._ import overflowdb.traversal.help import overflowdb.traversal.help.Doc +import scala.collection.IterableOnceOps import scala.jdk.CollectionConverters._ +trait MethodTraversalImplicits { + implicit def toMethodTraversalSingle[I <: nodes.Method](trav: I) = { + new MethodTraversal[I, SingleTravTypes.type](trav: Single[I]) + } + implicit def toMethodTraversalGeneric[I <: nodes.Method, T[_] <: SupportedTypes[_]](trav: TravTypesFor[T]#Collection[I]) = { + new MethodTraversal[I, TravTypesFor[T]](trav) + } + implicit def toMethodTraversalIterOnceOps[I <: nodes.Method, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { + new MethodTraversal[I, IterableOnceOpsTravTypes[CC, C]](trav) + } +} /** * A method, function, or procedure From d3f98e015e1ad6be64df34a12576fb369dd53494 Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Fri, 19 Nov 2021 16:19:46 +0100 Subject: [PATCH 15/27] Moved TravTypes classes. Reorganized implicits. --- performance/src/main/scala/MyTests.scala | 1 - .../semanticcpg/langv2/AnyTraversal.scala | 13 +++++ .../ExtensionClassImplicits.scala} | 6 +-- .../semanticcpg/langv2/TravTypes.scala | 36 ++++++++++++++ .../semanticcpg/langv2/package.scala | 47 +------------------ 5 files changed, 53 insertions(+), 50 deletions(-) rename semanticcpg/src/main/scala/io/shiftleft/semanticcpg/{language/New.scala => langv2/ExtensionClassImplicits.scala} (73%) create mode 100644 semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala diff --git a/performance/src/main/scala/MyTests.scala b/performance/src/main/scala/MyTests.scala index 36342e2f1..783d2a6d3 100644 --- a/performance/src/main/scala/MyTests.scala +++ b/performance/src/main/scala/MyTests.scala @@ -40,7 +40,6 @@ case class Bar[T](x: T) class MyTestNew { import io.shiftleft.semanticcpg.langv2._ - import io.shiftleft.semanticcpg.language.New._ @Benchmark def travNew(state: MyState) = { val x = state.method.methodReturn diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala index aa5002301..a67a3f51d 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala @@ -2,6 +2,19 @@ package io.shiftleft.semanticcpg.langv2 import overflowdb.traversal.RepeatStepIterator +import scala.collection.IterableOnceOps + +trait AnyTraversalImplicits { + implicit def toAnyTraversalGeneric[I, T[_] <: SupportedTypes[_]](trav: TravTypesFor[T]#Collection[I]) = { + new AnyTraversal[I, TravTypesFor[T]](trav) + } + implicit def toAnyTraversalIterOnceOps[I, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { + new AnyTraversal[I, IterableOnceOpsTravTypes[CC, C]](trav) + } + implicit def toAnyTraversalInternal[I, TT <: TravTypes](trav: TT#Collection[I]) = { + new AnyTraversal[I, TT](trav) + } +} class AnyTraversal[I, TT <: TravTypes](val trav: TT#Collection[I]) extends AnyVal { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/ExtensionClassImplicits.scala similarity index 73% rename from semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala rename to semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/ExtensionClassImplicits.scala index 75fd0f6ba..9db990908 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/New.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/ExtensionClassImplicits.scala @@ -1,9 +1,9 @@ -package io.shiftleft.semanticcpg.language +package io.shiftleft.semanticcpg.langv2 import io.shiftleft.semanticcpg.langv2.types.expressions.generalizations.AstNodeTraversalImplicits import io.shiftleft.semanticcpg.langv2.types.structure.{LocalTraversalImplicits, MethodTraversalImplicits} -object New +trait ExtensionClassImplicits extends MethodTraversalImplicits with AstNodeTraversalImplicits - with LocalTraversalImplicits \ No newline at end of file + with LocalTraversalImplicits diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala new file mode 100644 index 000000000..2eada1842 --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala @@ -0,0 +1,36 @@ +package io.shiftleft.semanticcpg.langv2 + +import scala.collection.IterableOnceOps + +trait TravTypes { + type Collection[_] + type CCOneToOne[_] + type CCOneToOption[_] + type CCOneToBoolean[_] + type CCOneToMany[_] +} + +object SingleTravTypes extends TravTypes { + override type Collection[T] = T + override type CCOneToOne[T] = T + override type CCOneToOption[T] = Option[T] + override type CCOneToBoolean[T] = Option[T] + override type CCOneToMany[T] = Seq[T] +} + +object OptionTravTypes extends TravTypes { + override type Collection[T] = Option[T] + override type CCOneToOne[T] = Option[T] + override type CCOneToOption[T] = Option[T] + override type CCOneToBoolean[T] = Option[T] + override type CCOneToMany[T] = Seq[T] +} + +class IterableOnceOpsTravTypes[CC[_], C] extends TravTypes { + override type Collection[T] = IterableOnceOps[T, CC, C] + override type CCOneToOne[T] = CC[T] + override type CCOneToOption[T] = CC[T] + override type CCOneToBoolean[T] = C + override type CCOneToMany[T] = CC[T] +} + diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala index db903d191..168e13227 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala @@ -1,42 +1,8 @@ package io.shiftleft.semanticcpg -import scala.collection.{IterableFactory, IterableOnceOps} - -package object langv2 extends InstanceOfOpsImplicits { +package object langv2 extends ExtensionClassImplicits with AnyTraversalImplicits with InstanceOfOpsImplicits { type Single[T] = T - trait TravTypes { - type Collection[_] - type CCOneToOne[_] - type CCOneToOption[_] - type CCOneToBoolean[_] - type CCOneToMany[_] - } - - object SingleTravTypes extends TravTypes { - override type Collection[T] = T - override type CCOneToOne[T] = T - override type CCOneToOption[T] = Option[T] - override type CCOneToBoolean[T] = Option[T] - override type CCOneToMany[T] = Seq[T] - } - - object OptionTravTypes extends TravTypes { - override type Collection[T] = Option[T] - override type CCOneToOne[T] = Option[T] - override type CCOneToOption[T] = Option[T] - override type CCOneToBoolean[T] = Option[T] - override type CCOneToMany[T] = Seq[T] - } - - class IterableOnceOpsTravTypes[CC[_], C] extends TravTypes { - override type Collection[T] = IterableOnceOps[T, CC, C] - override type CCOneToOne[T] = CC[T] - override type CCOneToOption[T] = CC[T] - override type CCOneToBoolean[T] = C - override type CCOneToMany[T] = CC[T] - } - type SupportedTypes[T] = Option[T] type TravTypesFor[T[_]] = OptionTravTypes.type @@ -45,15 +11,4 @@ package object langv2 extends InstanceOfOpsImplicits { private val it2Ops = new IterableOnceOpsOps() implicit def toIt2Ops[CC[_], C] = it2Ops.asInstanceOf[IterableOnceOpsOps[CC, C]] - - implicit def toAnyTraversalNew1[I, T[_] <: SupportedTypes[_]](trav: TravTypesFor[T]#Collection[I]) = { - new AnyTraversal[I, TravTypesFor[T]](trav) - } - implicit def toAnyTraversalNew2[I, TT <: TravTypes](trav: TT#Collection[I]) = { - new AnyTraversal[I, TT](trav) - } - implicit def toAnyTraversalNew3[I, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - new AnyTraversal[I, IterableOnceOpsTravTypes[CC, C]](trav) - } - } From aef9e7b6c1010b930965eac7df2ff182906bf162 Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Fri, 19 Nov 2021 16:50:06 +0100 Subject: [PATCH 16/27] Test adjustment. --- performance/src/main/scala/MyTests.scala | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/performance/src/main/scala/MyTests.scala b/performance/src/main/scala/MyTests.scala index 783d2a6d3..72df1c947 100644 --- a/performance/src/main/scala/MyTests.scala +++ b/performance/src/main/scala/MyTests.scala @@ -52,12 +52,6 @@ class MyTestNew { x } - @Benchmark - def refIdNew1(state: MyState) = { - val x = toLocalNew(state.local).referencingIdentifiers - x - } - @Benchmark def refIdNew2(state: MyState) = { val x = toLocalTraversalSingle(state.local).referencingIdentifiers @@ -66,13 +60,13 @@ class MyTestNew { @Benchmark def refIdNew3(state: MyState) = { - val x = rftoSingleExt(state.local).referencingIdentifiers + val x = Seq.from(rftoSingleExt(state.local).referencingIdentifiers) x } @Benchmark def refIdBase(state: MyState) = { - val x = Iterable.from(state.local._refIn.asScala).filter(_.label == NodeTypes.IDENTIFIER) + val x = Seq.from(state.local._refIn.asScala.filter(_.label == NodeTypes.IDENTIFIER)) x } @@ -151,7 +145,7 @@ class MyTestsOld { @Benchmark def refIdOld(state: MyState) = { - val x = Traversal(state.local).referencingIdentifiers2.head + val x = Traversal(state.local).referencingIdentifiers2.toSeq x } From 6130f7fb2bd2e30ff762b1427a468b171f413b16 Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Fri, 19 Nov 2021 16:54:14 +0100 Subject: [PATCH 17/27] Make TravTypes abstract final classes. --- performance/src/main/scala/some/SomeDomain.scala | 2 +- .../scala/io/shiftleft/semanticcpg/langv2/TravOps.scala | 4 ++-- .../scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala | 6 +++--- .../scala/io/shiftleft/semanticcpg/langv2/package.scala | 2 +- .../expressions/generalizations/AstNodeTraversal.scala | 2 +- .../semanticcpg/langv2/types/structure/LocalTraversal.scala | 2 +- .../langv2/types/structure/MethodTraversal.scala | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/performance/src/main/scala/some/SomeDomain.scala b/performance/src/main/scala/some/SomeDomain.scala index 68594fdf6..4fd19e9bc 100644 --- a/performance/src/main/scala/some/SomeDomain.scala +++ b/performance/src/main/scala/some/SomeDomain.scala @@ -6,7 +6,7 @@ import scala.collection.IterableOnceOps object SomeDomain { implicit def toSynth1(p: D1) = { - new SynthExt[SingleTravTypes.type](p: Single[D1]) + new SynthExt[SingleTravTypes](p: Single[D1]) } implicit def toSynth2[T[_]](trav: TravTypesFor[T]#Collection[D1]) = { new SynthExt[TravTypesFor[T]](trav) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala index 43c5b6f07..264d3d5a4 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala @@ -18,7 +18,7 @@ trait TravOps[_TT <: TravTypes] { def oneToMany[I, O](trav: TT#Collection[I])(f: I => Iterator[O]): TT#CCOneToMany[O] } -object SingleOps extends TravOps[SingleTravTypes.type] { +object SingleOps extends TravOps[SingleTravTypes] { override def oneToOne[I, O](trav: this.type#TT#Collection[I])(f: I => O): this.type#TT#CCOneToOne[O] = { f(trav) @@ -41,7 +41,7 @@ object SingleOps extends TravOps[SingleTravTypes.type] { } } -object OptionOps extends TravOps[OptionTravTypes.type] { +object OptionOps extends TravOps[OptionTravTypes] { override def oneToOne[I, O](trav: this.type#TT#Collection[I])(f: I => O): this.type#TT#CCOneToOne[O] = { trav.map(f) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala index 2eada1842..eea117959 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala @@ -10,7 +10,7 @@ trait TravTypes { type CCOneToMany[_] } -object SingleTravTypes extends TravTypes { +abstract final class SingleTravTypes extends TravTypes { override type Collection[T] = T override type CCOneToOne[T] = T override type CCOneToOption[T] = Option[T] @@ -18,7 +18,7 @@ object SingleTravTypes extends TravTypes { override type CCOneToMany[T] = Seq[T] } -object OptionTravTypes extends TravTypes { +abstract final class OptionTravTypes extends TravTypes { override type Collection[T] = Option[T] override type CCOneToOne[T] = Option[T] override type CCOneToOption[T] = Option[T] @@ -26,7 +26,7 @@ object OptionTravTypes extends TravTypes { override type CCOneToMany[T] = Seq[T] } -class IterableOnceOpsTravTypes[CC[_], C] extends TravTypes { +abstract final class IterableOnceOpsTravTypes[CC[_], C] extends TravTypes { override type Collection[T] = IterableOnceOps[T, CC, C] override type CCOneToOne[T] = CC[T] override type CCOneToOption[T] = CC[T] diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala index 168e13227..5a9adbd44 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala @@ -4,7 +4,7 @@ package object langv2 extends ExtensionClassImplicits with AnyTraversalImplicits type Single[T] = T type SupportedTypes[T] = Option[T] - type TravTypesFor[T[_]] = OptionTravTypes.type + type TravTypesFor[T[_]] = OptionTravTypes implicit val singleOps = SingleOps implicit val optionOps = OptionOps diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala index f518da80c..4401b649a 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala @@ -10,7 +10,7 @@ import scala.jdk.CollectionConverters._ trait AstNodeTraversalImplicits { implicit def toAstNodeTraversalSingle[I <: AstNode](trav: I) = { - new AstNodeTraversal[I, SingleTravTypes.type](trav: Single[I]) + new AstNodeTraversal[I, SingleTravTypes](trav: Single[I]) } implicit def toAstNodeTraversalGeneric[I <: AstNode, T[_] <: SupportedTypes[_]](trav: TravTypesFor[T]#Collection[I]) = { new AstNodeTraversal[I, TravTypesFor[T]](trav) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala index 3cc5b5df7..3b5e767bf 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala @@ -10,7 +10,7 @@ import scala.jdk.CollectionConverters._ trait LocalTraversalImplicits { implicit def toLocalTraversalSingle[I <: nodes.Local](trav: I) = { - new LocalTraversal[I, SingleTravTypes.type](trav: Single[I]) + new LocalTraversal[I, SingleTravTypes](trav: Single[I]) } implicit def toLocalTraversalGeneric[I <: nodes.Local, T[_] <: SupportedTypes[_]](trav: TravTypesFor[T]#Collection[I]) = { new LocalTraversal[I, TravTypesFor[T]](trav) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala index baf3c81ea..d7d27e8f0 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala @@ -10,7 +10,7 @@ import scala.jdk.CollectionConverters._ trait MethodTraversalImplicits { implicit def toMethodTraversalSingle[I <: nodes.Method](trav: I) = { - new MethodTraversal[I, SingleTravTypes.type](trav: Single[I]) + new MethodTraversal[I, SingleTravTypes](trav: Single[I]) } implicit def toMethodTraversalGeneric[I <: nodes.Method, T[_] <: SupportedTypes[_]](trav: TravTypesFor[T]#Collection[I]) = { new MethodTraversal[I, TravTypesFor[T]](trav) From 4a9f7d5ba6448506ed3ff0f4a34cae9cd121bbaf Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Mon, 22 Nov 2021 20:17:51 +0100 Subject: [PATCH 18/27] tmp --- performance/src/main/scala/MyTests.scala | 2 +- .../src/main/scala/some/SomeDomain.scala | 52 +++++++++++++++---- .../semanticcpg/langv2/Example.scala | 51 ++++++++++++++++++ .../semanticcpg/langv2/TravTypes.scala | 6 +-- 4 files changed, 97 insertions(+), 14 deletions(-) create mode 100644 semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/Example.scala diff --git a/performance/src/main/scala/MyTests.scala b/performance/src/main/scala/MyTests.scala index 72df1c947..56d38739d 100644 --- a/performance/src/main/scala/MyTests.scala +++ b/performance/src/main/scala/MyTests.scala @@ -129,7 +129,7 @@ class MyTestNew { } def compileTest(state: MyState) = { - val c = toSynth3(Array(state.d1).view.slice(1,2)).toD2 + val c = toSynthIter(Array(state.d1).view.slice(1,2)).toD2 val d: View[D2] = c } } diff --git a/performance/src/main/scala/some/SomeDomain.scala b/performance/src/main/scala/some/SomeDomain.scala index 4fd19e9bc..42d92f93a 100644 --- a/performance/src/main/scala/some/SomeDomain.scala +++ b/performance/src/main/scala/some/SomeDomain.scala @@ -1,20 +1,23 @@ package some import io.shiftleft.semanticcpg.langv2._ +import some.SomeDomain.D1 import scala.collection.IterableOnceOps + object SomeDomain { - implicit def toSynth1(p: D1) = { - new SynthExt[SingleTravTypes](p: Single[D1]) - } - implicit def toSynth2[T[_]](trav: TravTypesFor[T]#Collection[D1]) = { - new SynthExt[TravTypesFor[T]](trav) - } - implicit def toSynth3[CC[_], C](trav: IterableOnceOps[D1, CC, C]) = { - //new AstTraversalNew[I, ({type X[B] = IterableOnceOps[B, CC, C]})#X, CC, CC, ({type X[B] = C})#X, CC](trav) - new SynthExt[IterableOnceOpsTravTypes[CC, C]](trav) - } + implicit def toSynthSingle[I <: D1](p: I) = { + new SynthExt[SingleTravTypes](p: Single[I]) + } + + implicit def toSynthOption[I <: D1](trav: Option[I]) = { + new SynthExt[OptionTravTypes](trav) + } + + implicit def toSynthIter[I <: D1, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { + new SynthExt[IterableOnceOpsTravTypes[CC, C]](trav) + } class SynthExt[TT <: TravTypes](val trav: TT#Collection[D1]) extends AnyVal { def toD2(implicit ops: TravOps[TT]) = { @@ -29,3 +32,32 @@ object SomeDomain { case class D2() } +/* +object SomeDomain2 { + trait SyncExtImplicits { + implicit def toSynthExtSingle(p: D1) = { + new SynthExt[Single, Single, Option, Option, Seq](p: Single[D1]) + } + + implicit def toSynthExtOption[T[_]](trav: Option[D1]) = { + new SynthExt[Option, Option, Option, Option, Seq](trav) + } + + implicit def toSynthExtIter[CC[_], C](trav: IterableOnceOps[D1, CC, C]) + : SynthExt[({type X[B] = IterableOnceOps[B, CC, C]})#X, CC, CC, ({type X[B] = C})#X, CC] = { + new SynthExt[({type X[B] = IterableOnceOps[B, CC, C]})#X, CC, CC, ({type X[B] = C})#X, CC](trav) + } + } + + class SynthExt[Collection[_], CCOneToOne[_], CCOneToOption[_], CCOneToBoolean[_], CCOneToMany[_]] + (val trav: Collection[D1]) extends AnyVal { + def toD2(implicit ops: TravOpsToOne[Collection]) = { + trav.map(_.x) + } + + def toD2Multi(implicit ops: TravOpsToOne[Collection]) = { + trav.flatMap(Iterator.single) + } + } +} + */ \ No newline at end of file diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/Example.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/Example.scala new file mode 100644 index 000000000..bdb1f8046 --- /dev/null +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/Example.scala @@ -0,0 +1,51 @@ +package io.shiftleft.semanticcpg.langv2 + +object Example { + trait TypeMultiplexer { + type IN[_] + type OUT1[_] + type OUT2[_] + } + + abstract final class TypeGroup1 extends TypeMultiplexer { + override type IN[T] = Option[T] + override type OUT1[T] = Option[T] + override type OUT2[T] = Iterable[T] + } + + trait Ops[_TM <: TypeMultiplexer] { + type TM = _TM + def map[I, O](a: TM#IN[I])(f: I => O): TM#OUT1[O] + def flatMap[I, O](a: TM#IN[I])(f: I => Iterator[O]): TM#OUT2[O] + } + + implicit object TypeGroup1Ops extends Ops[TypeGroup1] { + override def map[I, O](a: this.TM#IN[I])(f: I => O): this.TM#OUT1[O] = { + a.map(f) + } + + override def flatMap[I, O](a: this.TM#IN[I])(f: I => Iterator[O]): this.TM#OUT2[O] = { + a.map(f.andThen(Iterable.from)).getOrElse(Iterable.empty) + } + } + + case class Dom1(x: Iterable[Dom2]) + case class Dom2() + + implicit def toSomeExtension(i: Option[Dom1]) = { + new SomeExtension[TypeGroup1](i) + } + + class SomeExtension[TM <: TypeMultiplexer](val a: TM#IN[Dom1]) extends AnyVal { + def toFirstDom2(implicit ops: Ops[TM]) = { + ops.map(a)(_.x.head) + } + def toAllDom2(implicit ops: Ops[TM]) = { + ops.flatMap(a)(_.x.iterator) + } + } + + val d1 = Dom1(Dom2():: Dom2():: Nil) + Option(d1).toFirstDom2 + +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala index eea117959..49bed8c54 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala @@ -10,7 +10,7 @@ trait TravTypes { type CCOneToMany[_] } -abstract final class SingleTravTypes extends TravTypes { +final class SingleTravTypes extends TravTypes { override type Collection[T] = T override type CCOneToOne[T] = T override type CCOneToOption[T] = Option[T] @@ -18,7 +18,7 @@ abstract final class SingleTravTypes extends TravTypes { override type CCOneToMany[T] = Seq[T] } -abstract final class OptionTravTypes extends TravTypes { +final class OptionTravTypes extends TravTypes { override type Collection[T] = Option[T] override type CCOneToOne[T] = Option[T] override type CCOneToOption[T] = Option[T] @@ -26,7 +26,7 @@ abstract final class OptionTravTypes extends TravTypes { override type CCOneToMany[T] = Seq[T] } -abstract final class IterableOnceOpsTravTypes[CC[_], C] extends TravTypes { +final class IterableOnceOpsTravTypes[CC[_], C] extends TravTypes { override type Collection[T] = IterableOnceOps[T, CC, C] override type CCOneToOne[T] = CC[T] override type CCOneToOption[T] = CC[T] From 910dedc1afad83f263420e0d9398319bae61c1af Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Tue, 23 Nov 2021 02:08:42 +0100 Subject: [PATCH 19/27] Next step. --- .../src/main/scala/some/SomeDomain.scala | 12 ++-- .../semanticcpg/langv2/AnyTraversal.scala | 33 +++++----- .../semanticcpg/langv2/TravOps.scala | 60 ++++++++++++------- .../semanticcpg/langv2/TravTypes.scala | 36 ++--------- .../generalizations/AstNodeTraversal.scala | 14 ++--- .../types/structure/LocalTraversal.scala | 12 ++-- .../types/structure/MethodTraversal.scala | 14 ++--- 7 files changed, 85 insertions(+), 96 deletions(-) diff --git a/performance/src/main/scala/some/SomeDomain.scala b/performance/src/main/scala/some/SomeDomain.scala index 42d92f93a..6956530da 100644 --- a/performance/src/main/scala/some/SomeDomain.scala +++ b/performance/src/main/scala/some/SomeDomain.scala @@ -8,22 +8,22 @@ import scala.collection.IterableOnceOps object SomeDomain { implicit def toSynthSingle[I <: D1](p: I) = { - new SynthExt[SingleTravTypes](p: Single[I]) + new SynthExt[I, Single, SingleMarker](p: Single[I]) } implicit def toSynthOption[I <: D1](trav: Option[I]) = { - new SynthExt[OptionTravTypes](trav) + new SynthExt[I, Option, OptionMarker](trav) } implicit def toSynthIter[I <: D1, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - new SynthExt[IterableOnceOpsTravTypes[CC, C]](trav) + new SynthExt[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterMarker[CC, C]](trav) } - class SynthExt[TT <: TravTypes](val trav: TT#Collection[D1]) extends AnyVal { - def toD2(implicit ops: TravOps[TT]) = { + class SynthExt[I <: D1, IT[_], Marker](val trav: IT[D1]) extends AnyVal { + def toD2(implicit ops: TravOps[IT, Marker]) = { trav.map(_.x) } - def toD2Multi(implicit ops: TravOps[TT]) = { + def toD2Multi(implicit ops: TravOps[IT, Marker]) = { trav.flatMap(Iterator.single) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala index a67a3f51d..2da1b8d6b 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala @@ -5,60 +5,57 @@ import overflowdb.traversal.RepeatStepIterator import scala.collection.IterableOnceOps trait AnyTraversalImplicits { - implicit def toAnyTraversalGeneric[I, T[_] <: SupportedTypes[_]](trav: TravTypesFor[T]#Collection[I]) = { - new AnyTraversal[I, TravTypesFor[T]](trav) - } implicit def toAnyTraversalIterOnceOps[I, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - new AnyTraversal[I, IterableOnceOpsTravTypes[CC, C]](trav) + new AnyTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterMarker[CC, C]](trav) } - implicit def toAnyTraversalInternal[I, TT <: TravTypes](trav: TT#Collection[I]) = { - new AnyTraversal[I, TT](trav) + implicit def toAnyTraversalInternal[I, IT[_], Marker](trav: IT[I]) = { + new AnyTraversal[I, IT, Marker](trav) } } -class AnyTraversal[I, TT <: TravTypes](val trav: TT#Collection[I]) extends AnyVal { +class AnyTraversal[I, IT[_], Marker](val trav: IT[I]) extends AnyVal { - def cast[A]: TT#CCOneToOne[A] = { - trav.asInstanceOf[TT#CCOneToOne[A]] + def cast[A](implicit ops: TravOps[IT, Marker]): ops.CCOneToOne[A] = { + ops.cast(trav) } - def map[O](f: I => O)(implicit ops: TravOps[TT]): TT#CCOneToOne[O] = { + def map[O](f: I => O)(implicit ops: TravOps[IT, Marker]): ops.CCOneToOne[O] = { ops.oneToOne(trav)(f) } - def filter(f: I => Boolean)(implicit ops: TravOps[TT]): TT#CCOneToBoolean[I] = { + def filter(f: I => Boolean)(implicit ops: TravOps[IT, Marker]): ops.CCOneToBoolean[I] = { ops.oneToBoolean(trav)(f) } - def flatMap[O](f: I => Iterator[O])(implicit ops: TravOps[TT]): TT#CCOneToMany[O] = { + def flatMap[O](f: I => Iterator[O])(implicit ops: TravOps[IT, Marker]): ops.CCOneToMany[O] = { ops.oneToMany(trav)(f) } // We dont use ClassTag and instead use our own IsInstanceOfOps for performance reasons. - def collectAll[T](implicit ops: TravOps[TT], isInstanceOfOps: IsInstanceOfOps[T]): TT#CCOneToBoolean[T] = { - ops.oneToBoolean(trav)(isInstanceOfOps).asInstanceOf[TT#CCOneToBoolean[T]] + def collectAll[T](implicit ops: TravOps[IT, Marker], isInstanceOfOps: IsInstanceOfOps[T]): ops.CCOneToBoolean[T] = { + ops.oneToBoolean(trav)(isInstanceOfOps).asInstanceOf[ops.CCOneToBoolean[T]] } def rMap[O <: I](f: I => O, g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[TT]): TT#CCOneToMany[I] = { + implicit ops: TravOps[IT, Marker]): ops.CCOneToMany[I] = { val behaviour = g(new RepeatBehaviourBuilder()).build ops.oneToMany(trav)(i => new RepeatStepIterator(i, f.andThen(Iterator.single), behaviour)) } def rFlatMap[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[TT]): TT#CCOneToMany[I] = { + implicit ops: TravOps[IT, Marker]): ops.CCOneToMany[I] = { val behaviour = g(new RepeatBehaviourBuilder()).build ops.oneToMany(trav)(i => new RepeatStepIterator(i, f, behaviour)) } @deprecated("Use rFlatMap instead") def repeat[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[TT]): TT#CCOneToMany[I] = { + implicit ops: TravOps[IT, Marker]): ops.CCOneToMany[I] = { rFlatMap(f, g) } @deprecated("Calls to this function can be omitted") - def l: TT#Collection[I] = { + def l: IT[I] = { trav } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala index 264d3d5a4..269bbfade 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala @@ -10,25 +10,35 @@ import scala.collection.IterableOnceOps * @tparam FlatMapTraversal Flat map operation output traversal type. * Abbreviated in other generics as FT. */ -trait TravOps[_TT <: TravTypes] { - type TT = _TT - def oneToOne[I, O](trav: TT#Collection[I])(f: I => O): TT#CCOneToOne[O] - def oneToOption[I, O](trav: TT#Collection[I])(f: I => Option[O]): TT#CCOneToOption[O] - def oneToBoolean[I](trav: TT#Collection[I])(f: I => Boolean): TT#CCOneToBoolean[I] - def oneToMany[I, O](trav: TT#Collection[I])(f: I => Iterator[O]): TT#CCOneToMany[O] +trait TravOps[_Collection[_], Marker] { + type Collection[T] = _Collection[T] + type CCOneToOne[_] + type CCOneToOption[_] + type CCOneToBoolean[_] + type CCOneToMany[_] + + def oneToOne[I, O](trav: Collection[I])(f: I => O): CCOneToOne[O] + def oneToOption[I, O](trav: Collection[I])(f: I => Option[O]): CCOneToOption[O] + def oneToBoolean[I](trav: Collection[I])(f: I => Boolean): CCOneToBoolean[I] + def oneToMany[I, O](trav: Collection[I])(f: I => Iterator[O]): CCOneToMany[O] + def cast[T](trav: Collection[_]): CCOneToOne[T] = trav.asInstanceOf[CCOneToOne[T]] } -object SingleOps extends TravOps[SingleTravTypes] { +object SingleOps extends TravOps[Single, SingleMarker] { + override type CCOneToOne[T] = T + override type CCOneToOption[T] = Option[T] + override type CCOneToBoolean[T] = Option[T] + override type CCOneToMany[T] = Seq[T] - override def oneToOne[I, O](trav: this.type#TT#Collection[I])(f: I => O): this.type#TT#CCOneToOne[O] = { + override def oneToOne[I, O](trav: this.Collection[I])(f: I => O): CCOneToOne[O] = { f(trav) } - override def oneToOption[I, O](trav: this.type#TT#Collection[I])(f: I => Option[O]): this.type#TT#CCOneToOption[O] = { + override def oneToOption[I, O](trav: this.Collection[I])(f: I => Option[O]): CCOneToOption[O] = { f(trav) } - override def oneToBoolean[I](trav: this.type#TT#Collection[I])(f: I => Boolean): this.type#TT#CCOneToBoolean[I] = { + override def oneToBoolean[I](trav: this.Collection[I])(f: I => Boolean): CCOneToBoolean[I] = { if (f(trav)) { Some(trav) } else { @@ -36,26 +46,30 @@ object SingleOps extends TravOps[SingleTravTypes] { } } - override def oneToMany[I, O](trav: this.type#TT#Collection[I])(f: I => Iterator[O]): this.type#TT#CCOneToMany[O] = { + override def oneToMany[I, O](trav: this.Collection[I])(f: I => Iterator[O]): CCOneToMany[O] = { Seq.from(f(trav)) } } -object OptionOps extends TravOps[OptionTravTypes] { +object OptionOps extends TravOps[Option, OptionMarker] { + override type CCOneToOne[T] = Option[T] + override type CCOneToOption[T] = Option[T] + override type CCOneToBoolean[T] = Option[T] + override type CCOneToMany[T] = Seq[T] - override def oneToOne[I, O](trav: this.type#TT#Collection[I])(f: I => O): this.type#TT#CCOneToOne[O] = { + override def oneToOne[I, O](trav: this.Collection[I])(f: I => O): CCOneToOne[O] = { trav.map(f) } - override def oneToOption[I, O](trav: this.type#TT#Collection[I])(f: I => Option[O]): this.type#TT#CCOneToOption[O] = { + override def oneToOption[I, O](trav: this.Collection[I])(f: I => Option[O]): CCOneToOption[O] = { trav.flatMap(f) } - override def oneToBoolean[I](trav: this.type#TT#Collection[I])(f: I => Boolean): this.type#TT#CCOneToBoolean[I] = { + override def oneToBoolean[I](trav: this.Collection[I])(f: I => Boolean): CCOneToBoolean[I] = { trav.filter(f) } - override def oneToMany[I, O](trav: this.type#TT#Collection[I])(f: I => Iterator[O]): this.type#TT#CCOneToMany[O] = { + override def oneToMany[I, O](trav: this.Collection[I])(f: I => Iterator[O]): CCOneToMany[O] = { trav match { case Some(t) => Seq.from(f(t)) @@ -65,21 +79,25 @@ object OptionOps extends TravOps[OptionTravTypes] { } } -class IterableOnceOpsOps[CC[_], C] extends TravOps[IterableOnceOpsTravTypes[CC, C] ] { +class IterableOnceOpsOps[CC[_], C] extends TravOps[({type X[A] = IterableOnceOps[A, CC, C]})#X, IterMarker[CC, C]] { + override type CCOneToOne[T] = CC[T] + override type CCOneToOption[T] = CC[T] + override type CCOneToBoolean[T] = C + override type CCOneToMany[T] = CC[T] - override def oneToOne[I, O](trav: this.type#TT#Collection[I])(f: I => O): this.type#TT#CCOneToOne[O] = { + override def oneToOne[I, O](trav: this.Collection[I])(f: I => O): CCOneToOne[O] = { trav.map(f) } - override def oneToOption[I, O](trav: this.type#TT#Collection[I])(f: I => Option[O]): this.type#TT#CCOneToOption[O] = { + override def oneToOption[I, O](trav: this.Collection[I])(f: I => Option[O]): CCOneToOption[O] = { trav.flatMap(f) } - override def oneToBoolean[I](trav: this.type#TT#Collection[I])(f: I => Boolean): this.type#TT#CCOneToBoolean[I] = { + override def oneToBoolean[I](trav: this.Collection[I])(f: I => Boolean): CCOneToBoolean[I] = { trav.filter(f) } - override def oneToMany[I, O](trav: this.type#TT#Collection[I])(f: I => Iterator[O]): this.type#TT#CCOneToMany[O] = { + override def oneToMany[I, O](trav: this.Collection[I])(f: I => Iterator[O]): CCOneToMany[O] = { trav.flatMap(f) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala index 49bed8c54..f0afc20fa 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala @@ -2,35 +2,9 @@ package io.shiftleft.semanticcpg.langv2 import scala.collection.IterableOnceOps -trait TravTypes { - type Collection[_] - type CCOneToOne[_] - type CCOneToOption[_] - type CCOneToBoolean[_] - type CCOneToMany[_] +class SingleMarker +class OptionMarker +class IterMarker[_CC[_], _C] { + type CC[T] = _CC[T] + type C = _C } - -final class SingleTravTypes extends TravTypes { - override type Collection[T] = T - override type CCOneToOne[T] = T - override type CCOneToOption[T] = Option[T] - override type CCOneToBoolean[T] = Option[T] - override type CCOneToMany[T] = Seq[T] -} - -final class OptionTravTypes extends TravTypes { - override type Collection[T] = Option[T] - override type CCOneToOne[T] = Option[T] - override type CCOneToOption[T] = Option[T] - override type CCOneToBoolean[T] = Option[T] - override type CCOneToMany[T] = Seq[T] -} - -final class IterableOnceOpsTravTypes[CC[_], C] extends TravTypes { - override type Collection[T] = IterableOnceOps[T, CC, C] - override type CCOneToOne[T] = CC[T] - override type CCOneToOption[T] = CC[T] - override type CCOneToBoolean[T] = C - override type CCOneToMany[T] = CC[T] -} - diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala index 4401b649a..094bddf64 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala @@ -10,31 +10,31 @@ import scala.jdk.CollectionConverters._ trait AstNodeTraversalImplicits { implicit def toAstNodeTraversalSingle[I <: AstNode](trav: I) = { - new AstNodeTraversal[I, SingleTravTypes](trav: Single[I]) + new AstNodeTraversal[I, Single, SingleMarker](trav: Single[I]) } - implicit def toAstNodeTraversalGeneric[I <: AstNode, T[_] <: SupportedTypes[_]](trav: TravTypesFor[T]#Collection[I]) = { - new AstNodeTraversal[I, TravTypesFor[T]](trav) + implicit def toAstNodeTraversalGeneric[I <: AstNode](trav: Option[I]) = { + new AstNodeTraversal[I, Option, OptionMarker](trav) } implicit def toAstNodeTraversalIterOnceOps[I <: AstNode, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - new AstNodeTraversal[I, IterableOnceOpsTravTypes[CC, C]](trav) + new AstNodeTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterMarker[CC, C]](trav) } } -class AstNodeTraversal[I <: AstNode, TT <: TravTypes](val trav: TT#Collection[I]) +class AstNodeTraversal[I <: AstNode, IT[_], Marker](val trav: IT[I]) extends AnyVal { /** * Nodes of the AST rooted in this node, including the node itself. * */ @Doc("All nodes of the abstract syntax tree") - def ast(implicit ops1: TravOps[TT]) = + def ast(implicit ops1: TravOps[IT, Marker]) = trav.rFlatMap(_._astOut.asScala.asInstanceOf[Iterator[I]], _.emit) /** * Traverse only to AST nodes that are expressions * */ - def isExpression(implicit ops1: TravOps[TT]) = + def isExpression(implicit ops1: TravOps[IT, Marker]) = trav.collectAll[Expression] } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala index 3b5e767bf..3baf51062 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala @@ -10,18 +10,18 @@ import scala.jdk.CollectionConverters._ trait LocalTraversalImplicits { implicit def toLocalTraversalSingle[I <: nodes.Local](trav: I) = { - new LocalTraversal[I, SingleTravTypes](trav: Single[I]) + new LocalTraversal[I, Single, SingleMarker](trav: Single[I]) } - implicit def toLocalTraversalGeneric[I <: nodes.Local, T[_] <: SupportedTypes[_]](trav: TravTypesFor[T]#Collection[I]) = { - new LocalTraversal[I, TravTypesFor[T]](trav) + implicit def toLocalTraversalGeneric[I <: nodes.Local](trav: Option[I]) = { + new LocalTraversal[I, Option, OptionMarker](trav) } implicit def toLocalTraversalIterOnceOps[I <: nodes.Local, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - new LocalTraversal[I, IterableOnceOpsTravTypes[CC, C]](trav) + new LocalTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterMarker[CC, C]](trav) } } -class LocalTraversal[I <: nodes.Local, TT <: TravTypes](val trav: TT#Collection[I]) extends AnyVal { - def referencingIdentifiers(implicit ops1: TravOps[TT]): TT#CCOneToMany[Identifier] = { +class LocalTraversal[I <: nodes.Local, IT[_], Marker](val trav: IT[I]) extends AnyVal { + def referencingIdentifiers(implicit ops1: TravOps[IT, Marker]): ops1.CCOneToMany[Identifier] = { trav.flatMap(_._refIn.asScala.filter(_.label == NodeTypes.IDENTIFIER) .cast[Identifier]) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala index d7d27e8f0..2ade6f1a7 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala @@ -10,13 +10,13 @@ import scala.jdk.CollectionConverters._ trait MethodTraversalImplicits { implicit def toMethodTraversalSingle[I <: nodes.Method](trav: I) = { - new MethodTraversal[I, SingleTravTypes](trav: Single[I]) + new MethodTraversal[I, Single, SingleMarker](trav: Single[I]) } - implicit def toMethodTraversalGeneric[I <: nodes.Method, T[_] <: SupportedTypes[_]](trav: TravTypesFor[T]#Collection[I]) = { - new MethodTraversal[I, TravTypesFor[T]](trav) + implicit def toMethodTraversalGeneric[I <: nodes.Method](trav: Option[I]) = { + new MethodTraversal[I, Option, OptionMarker](trav) } implicit def toMethodTraversalIterOnceOps[I <: nodes.Method, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - new MethodTraversal[I, IterableOnceOpsTravTypes[CC, C]](trav) + new MethodTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterMarker[CC, C]](trav) } } @@ -24,13 +24,13 @@ trait MethodTraversalImplicits { * A method, function, or procedure * */ @help.Traversal(elementType = classOf[nodes.Method]) -class MethodTraversal[I <: nodes.Method, TT <: TravTypes](val trav: TT#Collection[I]) extends AnyVal { +class MethodTraversal[I <: nodes.Method, IT[_], Marker](val trav: IT[I]) extends AnyVal { /** * Traverse to parameters of the method * */ @Doc("All parameters") - def parameter(implicit ops1: TravOps[TT]) = { + def parameter(implicit ops1: TravOps[IT, Marker]) = { ops1.oneToMany(trav)(_._astOut.asScala.collect { case par: nodes.MethodParameterIn => par }) } @@ -38,7 +38,7 @@ class MethodTraversal[I <: nodes.Method, TT <: TravTypes](val trav: TT#Collectio * Traverse to formal return parameter * */ @Doc("All formal return parameters") - def methodReturn(implicit ops1: TravOps[TT]) = { + def methodReturn(implicit ops1: TravOps[IT, Marker]) = { ops1.oneToOne(trav)(_._astOut.asScala.collectFirst { case ret: nodes.MethodReturn => ret }.get) } From 7bcc581c80d74716ec347a19e69dc0422a70de27 Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Tue, 23 Nov 2021 11:45:39 +0100 Subject: [PATCH 20/27] Continued. --- performance/src/main/scala/MyTests.scala | 4 +++- performance/src/main/scala/some/SomeDomain.scala | 8 ++++---- .../io/shiftleft/semanticcpg/langv2/AnyTraversal.scala | 2 +- .../io/shiftleft/semanticcpg/langv2/TravOps.scala | 5 ++--- .../io/shiftleft/semanticcpg/langv2/TravTypes.scala | 3 +-- .../io/shiftleft/semanticcpg/langv2/package.scala | 3 --- .../expressions/generalizations/AstNodeTraversal.scala | 6 +++--- .../langv2/types/structure/LocalTraversal.scala | 10 +++++----- .../langv2/types/structure/MethodTraversal.scala | 6 +++--- 9 files changed, 22 insertions(+), 25 deletions(-) diff --git a/performance/src/main/scala/MyTests.scala b/performance/src/main/scala/MyTests.scala index 56d38739d..697138a07 100644 --- a/performance/src/main/scala/MyTests.scala +++ b/performance/src/main/scala/MyTests.scala @@ -119,7 +119,7 @@ class MyTestNew { def syntheticIterableNew(state: MyState) = { //toSynth(state.d1:: Nil).toD2Multi //val y: TravOps[Iterable, IterableTypes[Iterable, Iterable[Any]]] = toIt2Ops - val x: List[D2] = List(state.d1).toD2(toIt2Ops) + val x: List[D2] = List(state.d1).toD2 x } @@ -129,6 +129,8 @@ class MyTestNew { } def compileTest(state: MyState) = { + val a: D2 = state.d1.toD2 + val b: Option[D2] = Option(state.d1).toD2 val c = toSynthIter(Array(state.d1).view.slice(1,2)).toD2 val d: View[D2] = c } diff --git a/performance/src/main/scala/some/SomeDomain.scala b/performance/src/main/scala/some/SomeDomain.scala index 6956530da..ffe249ace 100644 --- a/performance/src/main/scala/some/SomeDomain.scala +++ b/performance/src/main/scala/some/SomeDomain.scala @@ -8,18 +8,18 @@ import scala.collection.IterableOnceOps object SomeDomain { implicit def toSynthSingle[I <: D1](p: I) = { - new SynthExt[I, Single, SingleMarker](p: Single[I]) + new SynthExt[I, Single, DefaultMarker](p: Single[I]) } - implicit def toSynthOption[I <: D1](trav: Option[I]) = { - new SynthExt[I, Option, OptionMarker](trav) + implicit def toSynthOption[I <: D1, IT[_]](trav: IT[I]) = { + new SynthExt[I, IT, DefaultMarker](trav) } implicit def toSynthIter[I <: D1, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { new SynthExt[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterMarker[CC, C]](trav) } - class SynthExt[I <: D1, IT[_], Marker](val trav: IT[D1]) extends AnyVal { + class SynthExt[I <: D1, IT[_], Marker](val trav: IT[I]) extends AnyVal { def toD2(implicit ops: TravOps[IT, Marker]) = { trav.map(_.x) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala index 2da1b8d6b..420bbfef0 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala @@ -16,7 +16,7 @@ trait AnyTraversalImplicits { class AnyTraversal[I, IT[_], Marker](val trav: IT[I]) extends AnyVal { def cast[A](implicit ops: TravOps[IT, Marker]): ops.CCOneToOne[A] = { - ops.cast(trav) + trav.asInstanceOf[ops.CCOneToOne[A]] } def map[O](f: I => O)(implicit ops: TravOps[IT, Marker]): ops.CCOneToOne[O] = { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala index 269bbfade..c633bd63f 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala @@ -21,10 +21,9 @@ trait TravOps[_Collection[_], Marker] { def oneToOption[I, O](trav: Collection[I])(f: I => Option[O]): CCOneToOption[O] def oneToBoolean[I](trav: Collection[I])(f: I => Boolean): CCOneToBoolean[I] def oneToMany[I, O](trav: Collection[I])(f: I => Iterator[O]): CCOneToMany[O] - def cast[T](trav: Collection[_]): CCOneToOne[T] = trav.asInstanceOf[CCOneToOne[T]] } -object SingleOps extends TravOps[Single, SingleMarker] { +object SingleOps extends TravOps[Single, DefaultMarker] { override type CCOneToOne[T] = T override type CCOneToOption[T] = Option[T] override type CCOneToBoolean[T] = Option[T] @@ -51,7 +50,7 @@ object SingleOps extends TravOps[Single, SingleMarker] { } } -object OptionOps extends TravOps[Option, OptionMarker] { +object OptionOps extends TravOps[Option, DefaultMarker] { override type CCOneToOne[T] = Option[T] override type CCOneToOption[T] = Option[T] override type CCOneToBoolean[T] = Option[T] diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala index f0afc20fa..268515a84 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala @@ -2,8 +2,7 @@ package io.shiftleft.semanticcpg.langv2 import scala.collection.IterableOnceOps -class SingleMarker -class OptionMarker +class DefaultMarker class IterMarker[_CC[_], _C] { type CC[T] = _CC[T] type C = _C diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala index 5a9adbd44..cc1c15248 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala @@ -3,9 +3,6 @@ package io.shiftleft.semanticcpg package object langv2 extends ExtensionClassImplicits with AnyTraversalImplicits with InstanceOfOpsImplicits { type Single[T] = T - type SupportedTypes[T] = Option[T] - type TravTypesFor[T[_]] = OptionTravTypes - implicit val singleOps = SingleOps implicit val optionOps = OptionOps private val it2Ops = new IterableOnceOpsOps() diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala index 094bddf64..fe5d1bba4 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala @@ -10,10 +10,10 @@ import scala.jdk.CollectionConverters._ trait AstNodeTraversalImplicits { implicit def toAstNodeTraversalSingle[I <: AstNode](trav: I) = { - new AstNodeTraversal[I, Single, SingleMarker](trav: Single[I]) + new AstNodeTraversal[I, Single, DefaultMarker](trav: Single[I]) } - implicit def toAstNodeTraversalGeneric[I <: AstNode](trav: Option[I]) = { - new AstNodeTraversal[I, Option, OptionMarker](trav) + implicit def toAstNodeTraversalGeneric[I <: AstNode, IT[_] <: Option[_]](trav: IT[I]) = { + new AstNodeTraversal[I, IT, DefaultMarker](trav) } implicit def toAstNodeTraversalIterOnceOps[I <: AstNode, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { new AstNodeTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterMarker[CC, C]](trav) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala index 3baf51062..526850e46 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala @@ -8,14 +8,14 @@ import scala.collection.IterableOnceOps import scala.jdk.CollectionConverters._ trait LocalTraversalImplicits { - implicit def toLocalTraversalSingle[I <: nodes.Local](trav: I) = { - new LocalTraversal[I, Single, SingleMarker](trav: Single[I]) + new LocalTraversal[I, Single, DefaultMarker](trav: Single[I]) } - implicit def toLocalTraversalGeneric[I <: nodes.Local](trav: Option[I]) = { - new LocalTraversal[I, Option, OptionMarker](trav) + implicit def toLocalTraversalGeneric[I <: nodes.Local, IT[_] <: Option[_]](trav: IT[I]) = { + new LocalTraversal[I, IT, DefaultMarker](trav) } - implicit def toLocalTraversalIterOnceOps[I <: nodes.Local, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { + implicit def toLocalTraversalIterOnceOps[I <: nodes.Local, CC[_], C](trav: IterableOnceOps[I, CC, C]) + : LocalTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterMarker[CC, C]] = { new LocalTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterMarker[CC, C]](trav) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala index 2ade6f1a7..92386bfc2 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala @@ -10,10 +10,10 @@ import scala.jdk.CollectionConverters._ trait MethodTraversalImplicits { implicit def toMethodTraversalSingle[I <: nodes.Method](trav: I) = { - new MethodTraversal[I, Single, SingleMarker](trav: Single[I]) + new MethodTraversal[I, Single, DefaultMarker](trav: Single[I]) } - implicit def toMethodTraversalGeneric[I <: nodes.Method](trav: Option[I]) = { - new MethodTraversal[I, Option, OptionMarker](trav) + implicit def toMethodTraversalGeneric[I <: nodes.Method, IT[_] <: Option[_]](trav: IT[I]) = { + new MethodTraversal[I, IT, DefaultMarker](trav) } implicit def toMethodTraversalIterOnceOps[I <: nodes.Method, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { new MethodTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterMarker[CC, C]](trav) From a70ab0632fa57296d553a251b31690dfb908a667 Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Tue, 23 Nov 2021 12:15:54 +0100 Subject: [PATCH 21/27] Get rid of DefaultMarker and use Nothing instead. Always specify implicit return types. --- performance/src/main/scala/some/SomeDomain.scala | 12 ++++++------ .../shiftleft/semanticcpg/langv2/AnyTraversal.scala | 2 +- .../io/shiftleft/semanticcpg/langv2/TravOps.scala | 13 +++++++++---- .../io/shiftleft/semanticcpg/langv2/TravTypes.scala | 7 ------- .../generalizations/AstNodeTraversal.scala | 13 +++++++------ .../langv2/types/structure/LocalTraversal.scala | 12 ++++++------ .../langv2/types/structure/MethodTraversal.scala | 13 +++++++------ 7 files changed, 36 insertions(+), 36 deletions(-) diff --git a/performance/src/main/scala/some/SomeDomain.scala b/performance/src/main/scala/some/SomeDomain.scala index ffe249ace..e39ff2aa4 100644 --- a/performance/src/main/scala/some/SomeDomain.scala +++ b/performance/src/main/scala/some/SomeDomain.scala @@ -7,16 +7,16 @@ import scala.collection.IterableOnceOps object SomeDomain { - implicit def toSynthSingle[I <: D1](p: I) = { - new SynthExt[I, Single, DefaultMarker](p: Single[I]) + implicit def toSynthSingle[I <: D1](p: I): SynthExt[I, Single, Nothing] = { + new SynthExt(p: Single[I]) } - implicit def toSynthOption[I <: D1, IT[_]](trav: IT[I]) = { - new SynthExt[I, IT, DefaultMarker](trav) + implicit def toSynthOption[I <: D1, IT[_] <: Option[_]](trav: IT[I]): SynthExt[I, IT, Nothing] = { + new SynthExt(trav) } - implicit def toSynthIter[I <: D1, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - new SynthExt[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterMarker[CC, C]](trav) + implicit def toSynthIter[I <: D1, CC[_], C](trav: IterableOnceOps[I, CC, C]): SynthExt[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { + new SynthExt[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) } class SynthExt[I <: D1, IT[_], Marker](val trav: IT[I]) extends AnyVal { diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala index 420bbfef0..ca162f4e0 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala @@ -6,7 +6,7 @@ import scala.collection.IterableOnceOps trait AnyTraversalImplicits { implicit def toAnyTraversalIterOnceOps[I, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - new AnyTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterMarker[CC, C]](trav) + new AnyTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) } implicit def toAnyTraversalInternal[I, IT[_], Marker](trav: IT[I]) = { new AnyTraversal[I, IT, Marker](trav) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala index c633bd63f..6d3a94eec 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala @@ -10,7 +10,7 @@ import scala.collection.IterableOnceOps * @tparam FlatMapTraversal Flat map operation output traversal type. * Abbreviated in other generics as FT. */ -trait TravOps[_Collection[_], Marker] { +trait TravOps[_Collection[_], ExtraTypes] { type Collection[T] = _Collection[T] type CCOneToOne[_] type CCOneToOption[_] @@ -23,7 +23,7 @@ trait TravOps[_Collection[_], Marker] { def oneToMany[I, O](trav: Collection[I])(f: I => Iterator[O]): CCOneToMany[O] } -object SingleOps extends TravOps[Single, DefaultMarker] { +object SingleOps extends TravOps[Single, Nothing] { override type CCOneToOne[T] = T override type CCOneToOption[T] = Option[T] override type CCOneToBoolean[T] = Option[T] @@ -50,7 +50,7 @@ object SingleOps extends TravOps[Single, DefaultMarker] { } } -object OptionOps extends TravOps[Option, DefaultMarker] { +object OptionOps extends TravOps[Option, Nothing] { override type CCOneToOne[T] = Option[T] override type CCOneToOption[T] = Option[T] override type CCOneToBoolean[T] = Option[T] @@ -78,7 +78,12 @@ object OptionOps extends TravOps[Option, DefaultMarker] { } } -class IterableOnceOpsOps[CC[_], C] extends TravOps[({type X[A] = IterableOnceOps[A, CC, C]})#X, IterMarker[CC, C]] { +class IterTypes[_CC[_], _C] { + type CC[T] = _CC[T] + type C = _C +} + +class IterableOnceOpsOps[CC[_], C] extends TravOps[({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] { override type CCOneToOne[T] = CC[T] override type CCOneToOption[T] = CC[T] override type CCOneToBoolean[T] = C diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala index 268515a84..071580c9e 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravTypes.scala @@ -1,9 +1,2 @@ package io.shiftleft.semanticcpg.langv2 -import scala.collection.IterableOnceOps - -class DefaultMarker -class IterMarker[_CC[_], _C] { - type CC[T] = _CC[T] - type C = _C -} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala index fe5d1bba4..dee7a5837 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala @@ -9,14 +9,15 @@ import scala.jdk.CollectionConverters._ trait AstNodeTraversalImplicits { - implicit def toAstNodeTraversalSingle[I <: AstNode](trav: I) = { - new AstNodeTraversal[I, Single, DefaultMarker](trav: Single[I]) + implicit def toAstNodeTraversalSingle[I <: AstNode](trav: I): AstNodeTraversal[I, Single, Nothing] = { + new AstNodeTraversal(trav: Single[I]) } - implicit def toAstNodeTraversalGeneric[I <: AstNode, IT[_] <: Option[_]](trav: IT[I]) = { - new AstNodeTraversal[I, IT, DefaultMarker](trav) + implicit def toAstNodeTraversalGeneric[I <: AstNode, IT[_] <: Option[_]](trav: IT[I]): AstNodeTraversal[I, IT, Nothing] = { + new AstNodeTraversal(trav) } - implicit def toAstNodeTraversalIterOnceOps[I <: AstNode, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - new AstNodeTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterMarker[CC, C]](trav) + implicit def toAstNodeTraversalIterOnceOps[I <: AstNode, CC[_], C](trav: IterableOnceOps[I, CC, C]) + : AstNodeTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { + new AstNodeTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala index 526850e46..f1a84bbdd 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala @@ -8,15 +8,15 @@ import scala.collection.IterableOnceOps import scala.jdk.CollectionConverters._ trait LocalTraversalImplicits { - implicit def toLocalTraversalSingle[I <: nodes.Local](trav: I) = { - new LocalTraversal[I, Single, DefaultMarker](trav: Single[I]) + implicit def toLocalTraversalSingle[I <: nodes.Local](trav: I): LocalTraversal[I, Single, Nothing] = { + new LocalTraversal(trav: Single[I]) } - implicit def toLocalTraversalGeneric[I <: nodes.Local, IT[_] <: Option[_]](trav: IT[I]) = { - new LocalTraversal[I, IT, DefaultMarker](trav) + implicit def toLocalTraversalGeneric[I <: nodes.Local, IT[_] <: Option[_]](trav: IT[I]): LocalTraversal[I, IT, Nothing] = { + new LocalTraversal(trav) } implicit def toLocalTraversalIterOnceOps[I <: nodes.Local, CC[_], C](trav: IterableOnceOps[I, CC, C]) - : LocalTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterMarker[CC, C]] = { - new LocalTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterMarker[CC, C]](trav) + : LocalTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { + new LocalTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala index 92386bfc2..ed82f48c6 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala @@ -9,14 +9,15 @@ import scala.collection.IterableOnceOps import scala.jdk.CollectionConverters._ trait MethodTraversalImplicits { - implicit def toMethodTraversalSingle[I <: nodes.Method](trav: I) = { - new MethodTraversal[I, Single, DefaultMarker](trav: Single[I]) + implicit def toMethodTraversalSingle[I <: nodes.Method](trav: I): MethodTraversal[I, Single, Nothing] = { + new MethodTraversal(trav: Single[I]) } - implicit def toMethodTraversalGeneric[I <: nodes.Method, IT[_] <: Option[_]](trav: IT[I]) = { - new MethodTraversal[I, IT, DefaultMarker](trav) + implicit def toMethodTraversalGeneric[I <: nodes.Method, IT[_] <: Option[_]](trav: IT[I]): MethodTraversal[I, IT, Nothing] = { + new MethodTraversal(trav) } - implicit def toMethodTraversalIterOnceOps[I <: nodes.Method, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - new MethodTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterMarker[CC, C]](trav) + implicit def toMethodTraversalIterOnceOps[I <: nodes.Method, CC[_], C](trav: IterableOnceOps[I, CC, C]) + : MethodTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { + new MethodTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) } } From 2e9629cd122b33afc2d8300dbd96bd5f349658c6 Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Tue, 23 Nov 2021 15:35:10 +0100 Subject: [PATCH 22/27] Implicits directly on Option. --- performance/src/main/scala/some/SomeDomain.scala | 2 +- .../types/expressions/generalizations/AstNodeTraversal.scala | 2 +- .../semanticcpg/langv2/types/structure/LocalTraversal.scala | 2 +- .../semanticcpg/langv2/types/structure/MethodTraversal.scala | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/performance/src/main/scala/some/SomeDomain.scala b/performance/src/main/scala/some/SomeDomain.scala index e39ff2aa4..e01ca393e 100644 --- a/performance/src/main/scala/some/SomeDomain.scala +++ b/performance/src/main/scala/some/SomeDomain.scala @@ -11,7 +11,7 @@ object SomeDomain { new SynthExt(p: Single[I]) } - implicit def toSynthOption[I <: D1, IT[_] <: Option[_]](trav: IT[I]): SynthExt[I, IT, Nothing] = { + implicit def toSynthOption[I <: D1](trav: Option[I]): SynthExt[I, Option, Nothing] = { new SynthExt(trav) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala index dee7a5837..b0c927b1e 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala @@ -12,7 +12,7 @@ trait AstNodeTraversalImplicits { implicit def toAstNodeTraversalSingle[I <: AstNode](trav: I): AstNodeTraversal[I, Single, Nothing] = { new AstNodeTraversal(trav: Single[I]) } - implicit def toAstNodeTraversalGeneric[I <: AstNode, IT[_] <: Option[_]](trav: IT[I]): AstNodeTraversal[I, IT, Nothing] = { + implicit def toAstNodeTraversalGeneric[I <: AstNode](trav: Option[I]): AstNodeTraversal[I, Option, Nothing] = { new AstNodeTraversal(trav) } implicit def toAstNodeTraversalIterOnceOps[I <: AstNode, CC[_], C](trav: IterableOnceOps[I, CC, C]) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala index f1a84bbdd..17fcba413 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala @@ -11,7 +11,7 @@ trait LocalTraversalImplicits { implicit def toLocalTraversalSingle[I <: nodes.Local](trav: I): LocalTraversal[I, Single, Nothing] = { new LocalTraversal(trav: Single[I]) } - implicit def toLocalTraversalGeneric[I <: nodes.Local, IT[_] <: Option[_]](trav: IT[I]): LocalTraversal[I, IT, Nothing] = { + implicit def toLocalTraversalGeneric[I <: nodes.Local](trav: Option[I]): LocalTraversal[I, Option, Nothing] = { new LocalTraversal(trav) } implicit def toLocalTraversalIterOnceOps[I <: nodes.Local, CC[_], C](trav: IterableOnceOps[I, CC, C]) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala index ed82f48c6..845153403 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala @@ -12,7 +12,7 @@ trait MethodTraversalImplicits { implicit def toMethodTraversalSingle[I <: nodes.Method](trav: I): MethodTraversal[I, Single, Nothing] = { new MethodTraversal(trav: Single[I]) } - implicit def toMethodTraversalGeneric[I <: nodes.Method, IT[_] <: Option[_]](trav: IT[I]): MethodTraversal[I, IT, Nothing] = { + implicit def toMethodTraversalGeneric[I <: nodes.Method](trav: Option[I]): MethodTraversal[I, Option, Nothing] = { new MethodTraversal(trav) } implicit def toMethodTraversalIterOnceOps[I <: nodes.Method, CC[_], C](trav: IterableOnceOps[I, CC, C]) From d733696db6f9166f11da14a0a6c1730f4bf37eb6 Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Wed, 24 Nov 2021 12:17:39 +0100 Subject: [PATCH 23/27] Back to reality. --- performance/src/main/scala/MyTests.scala | 6 +- .../src/main/scala/some/SomeDomain.scala | 14 ++-- .../semanticcpg/langv2/AnyTraversal.scala | 25 +++---- .../semanticcpg/langv2/TravOps.scala | 68 ++++++++++++------- .../semanticcpg/langv2/package.scala | 5 +- .../generalizations/AstNodeTraversal.scala | 14 ++-- .../types/structure/LocalTraversal.scala | 12 ++-- .../types/structure/MethodTraversal.scala | 14 ++-- 8 files changed, 79 insertions(+), 79 deletions(-) diff --git a/performance/src/main/scala/MyTests.scala b/performance/src/main/scala/MyTests.scala index 697138a07..0b6fde73e 100644 --- a/performance/src/main/scala/MyTests.scala +++ b/performance/src/main/scala/MyTests.scala @@ -11,7 +11,7 @@ import org.openjdk.jmh.annotations._ import MyTests._ import io.shiftleft.semanticcpg.language.types.structure.LocalReferencingIdentifiers -import scala.collection.{View, mutable} +import scala.collection.{IndexedSeqView, View, mutable} import scala.collection.mutable.{ArrayBuffer, ListBuffer} import org.openjdk.jmh.infra.Blackhole import overflowdb.traversal.Traversal @@ -131,8 +131,8 @@ class MyTestNew { def compileTest(state: MyState) = { val a: D2 = state.d1.toD2 val b: Option[D2] = Option(state.d1).toD2 - val c = toSynthIter(Array(state.d1).view.slice(1,2)).toD2 - val d: View[D2] = c + val c = toSynthOption(Array(state.d1).view.slice(1,2)).toD2 + val d: IndexedSeqView[D2] = c } } diff --git a/performance/src/main/scala/some/SomeDomain.scala b/performance/src/main/scala/some/SomeDomain.scala index e01ca393e..4493898f5 100644 --- a/performance/src/main/scala/some/SomeDomain.scala +++ b/performance/src/main/scala/some/SomeDomain.scala @@ -7,23 +7,19 @@ import scala.collection.IterableOnceOps object SomeDomain { - implicit def toSynthSingle[I <: D1](p: I): SynthExt[I, Single, Nothing] = { + implicit def toSynthSingle[I <: D1](p: I): SynthExt[I, Single] = { new SynthExt(p: Single[I]) } - implicit def toSynthOption[I <: D1](trav: Option[I]): SynthExt[I, Option, Nothing] = { + implicit def toSynthOption[I <: D1, IT[_]](trav: IT[I]): SynthExt[I, IT] = { new SynthExt(trav) } - implicit def toSynthIter[I <: D1, CC[_], C](trav: IterableOnceOps[I, CC, C]): SynthExt[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { - new SynthExt[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) - } - - class SynthExt[I <: D1, IT[_], Marker](val trav: IT[I]) extends AnyVal { - def toD2(implicit ops: TravOps[IT, Marker]) = { + class SynthExt[I <: D1, IT[_]](val trav: IT[I]) extends AnyVal { + def toD2(implicit ops: TravOps[IT]) = { trav.map(_.x) } - def toD2Multi(implicit ops: TravOps[IT, Marker]) = { + def toD2Multi(implicit ops: TravOps[IT]) = { trav.flatMap(Iterator.single) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala index ca162f4e0..138edb75c 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala @@ -5,52 +5,49 @@ import overflowdb.traversal.RepeatStepIterator import scala.collection.IterableOnceOps trait AnyTraversalImplicits { - implicit def toAnyTraversalIterOnceOps[I, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { - new AnyTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) - } - implicit def toAnyTraversalInternal[I, IT[_], Marker](trav: IT[I]) = { - new AnyTraversal[I, IT, Marker](trav) + implicit def toAnyTraversalInternal[I, IT[_]](trav: IT[I]) = { + new AnyTraversal[I, IT](trav) } } -class AnyTraversal[I, IT[_], Marker](val trav: IT[I]) extends AnyVal { +class AnyTraversal[I, IT[_]](val trav: IT[I]) extends AnyVal { - def cast[A](implicit ops: TravOps[IT, Marker]): ops.CCOneToOne[A] = { + def cast[A](implicit ops: TravOps[IT]): ops.CCOneToOne[A] = { trav.asInstanceOf[ops.CCOneToOne[A]] } - def map[O](f: I => O)(implicit ops: TravOps[IT, Marker]): ops.CCOneToOne[O] = { + def map[O](f: I => O)(implicit ops: TravOps[IT]): ops.CCOneToOne[O] = { ops.oneToOne(trav)(f) } - def filter(f: I => Boolean)(implicit ops: TravOps[IT, Marker]): ops.CCOneToBoolean[I] = { + def filter(f: I => Boolean)(implicit ops: TravOps[IT]): ops.CCOneToBoolean[I] = { ops.oneToBoolean(trav)(f) } - def flatMap[O](f: I => Iterator[O])(implicit ops: TravOps[IT, Marker]): ops.CCOneToMany[O] = { + def flatMap[O](f: I => Iterator[O])(implicit ops: TravOps[IT]): ops.CCOneToMany[O] = { ops.oneToMany(trav)(f) } // We dont use ClassTag and instead use our own IsInstanceOfOps for performance reasons. - def collectAll[T](implicit ops: TravOps[IT, Marker], isInstanceOfOps: IsInstanceOfOps[T]): ops.CCOneToBoolean[T] = { + def collectAll[T](implicit ops: TravOps[IT], isInstanceOfOps: IsInstanceOfOps[T]): ops.CCOneToBoolean[T] = { ops.oneToBoolean(trav)(isInstanceOfOps).asInstanceOf[ops.CCOneToBoolean[T]] } def rMap[O <: I](f: I => O, g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[IT, Marker]): ops.CCOneToMany[I] = { + implicit ops: TravOps[IT]): ops.CCOneToMany[I] = { val behaviour = g(new RepeatBehaviourBuilder()).build ops.oneToMany(trav)(i => new RepeatStepIterator(i, f.andThen(Iterator.single), behaviour)) } def rFlatMap[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[IT, Marker]): ops.CCOneToMany[I] = { + implicit ops: TravOps[IT]): ops.CCOneToMany[I] = { val behaviour = g(new RepeatBehaviourBuilder()).build ops.oneToMany(trav)(i => new RepeatStepIterator(i, f, behaviour)) } @deprecated("Use rFlatMap instead") def repeat[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[IT, Marker]): ops.CCOneToMany[I] = { + implicit ops: TravOps[IT]): ops.CCOneToMany[I] = { rFlatMap(f, g) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala index 6d3a94eec..c33418fa9 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala @@ -10,7 +10,7 @@ import scala.collection.IterableOnceOps * @tparam FlatMapTraversal Flat map operation output traversal type. * Abbreviated in other generics as FT. */ -trait TravOps[_Collection[_], ExtraTypes] { +trait TravOps[_Collection[_]] { type Collection[T] = _Collection[T] type CCOneToOne[_] type CCOneToOption[_] @@ -23,21 +23,21 @@ trait TravOps[_Collection[_], ExtraTypes] { def oneToMany[I, O](trav: Collection[I])(f: I => Iterator[O]): CCOneToMany[O] } -object SingleOps extends TravOps[Single, Nothing] { +object SingleOps extends TravOps[Single] { override type CCOneToOne[T] = T override type CCOneToOption[T] = Option[T] override type CCOneToBoolean[T] = Option[T] override type CCOneToMany[T] = Seq[T] - override def oneToOne[I, O](trav: this.Collection[I])(f: I => O): CCOneToOne[O] = { + override def oneToOne[I, O](trav: Collection[I])(f: I => O): CCOneToOne[O] = { f(trav) } - override def oneToOption[I, O](trav: this.Collection[I])(f: I => Option[O]): CCOneToOption[O] = { + override def oneToOption[I, O](trav: Collection[I])(f: I => Option[O]): CCOneToOption[O] = { f(trav) } - override def oneToBoolean[I](trav: this.Collection[I])(f: I => Boolean): CCOneToBoolean[I] = { + override def oneToBoolean[I](trav: Collection[I])(f: I => Boolean): CCOneToBoolean[I] = { if (f(trav)) { Some(trav) } else { @@ -45,30 +45,30 @@ object SingleOps extends TravOps[Single, Nothing] { } } - override def oneToMany[I, O](trav: this.Collection[I])(f: I => Iterator[O]): CCOneToMany[O] = { + override def oneToMany[I, O](trav: Collection[I])(f: I => Iterator[O]): CCOneToMany[O] = { Seq.from(f(trav)) } } -object OptionOps extends TravOps[Option, Nothing] { +object OptionOps extends TravOps[Option] { override type CCOneToOne[T] = Option[T] override type CCOneToOption[T] = Option[T] override type CCOneToBoolean[T] = Option[T] override type CCOneToMany[T] = Seq[T] - override def oneToOne[I, O](trav: this.Collection[I])(f: I => O): CCOneToOne[O] = { + override def oneToOne[I, O](trav: Collection[I])(f: I => O): CCOneToOne[O] = { trav.map(f) } - override def oneToOption[I, O](trav: this.Collection[I])(f: I => Option[O]): CCOneToOption[O] = { + override def oneToOption[I, O](trav: Collection[I])(f: I => Option[O]): CCOneToOption[O] = { trav.flatMap(f) } - override def oneToBoolean[I](trav: this.Collection[I])(f: I => Boolean): CCOneToBoolean[I] = { + override def oneToBoolean[I](trav: Collection[I])(f: I => Boolean): CCOneToBoolean[I] = { trav.filter(f) } - override def oneToMany[I, O](trav: this.Collection[I])(f: I => Iterator[O]): CCOneToMany[O] = { + override def oneToMany[I, O](trav: Collection[I])(f: I => Iterator[O]): CCOneToMany[O] = { trav match { case Some(t) => Seq.from(f(t)) @@ -78,30 +78,48 @@ object OptionOps extends TravOps[Option, Nothing] { } } -class IterTypes[_CC[_], _C] { - type CC[T] = _CC[T] - type C = _C -} - -class IterableOnceOpsOps[CC[_], C] extends TravOps[({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] { - override type CCOneToOne[T] = CC[T] - override type CCOneToOption[T] = CC[T] - override type CCOneToBoolean[T] = C - override type CCOneToMany[T] = CC[T] +object IteratorOps extends TravOps[Iterator] { + override type CCOneToOne[T] = Iterator[T] + override type CCOneToOption[T] = Iterator[T] + override type CCOneToBoolean[T] = Iterator[T] + override type CCOneToMany[T] = Iterator[T] - override def oneToOne[I, O](trav: this.Collection[I])(f: I => O): CCOneToOne[O] = { + override def oneToOne[I, O](trav: Collection[I])(f: I => O): CCOneToOne[O] = { trav.map(f) } - override def oneToOption[I, O](trav: this.Collection[I])(f: I => Option[O]): CCOneToOption[O] = { + override def oneToOption[I, O](trav: Collection[I])(f: I => Option[O]): CCOneToOption[O] = { trav.flatMap(f) } - override def oneToBoolean[I](trav: this.Collection[I])(f: I => Boolean): CCOneToBoolean[I] = { + override def oneToBoolean[I](trav: Collection[I])(f: I => Boolean): CCOneToBoolean[I] = { trav.filter(f) } - override def oneToMany[I, O](trav: this.Collection[I])(f: I => Iterator[O]): CCOneToMany[O] = { + override def oneToMany[I, O](trav: Collection[I])(f: I => Iterator[O]): CCOneToMany[O] = { trav.flatMap(f) } } + +class IterableOps[IT[T] <: Iterable[T]] extends TravOps[IT] { + override type CCOneToOne[T] = IT[T] + override type CCOneToOption[T] = IT[T] + override type CCOneToBoolean[T] = IT[T] + override type CCOneToMany[T] = IT[T] + + override def oneToOne[I, O](trav: Collection[I])(f: I => O): CCOneToOne[O] = { + trav.map(f).asInstanceOf[IT[O]] + } + + override def oneToOption[I, O](trav: Collection[I])(f: I => Option[O]): CCOneToOption[O] = { + trav.flatMap(f).asInstanceOf[IT[O]] + } + + override def oneToBoolean[I](trav: Collection[I])(f: I => Boolean): CCOneToBoolean[I] = { + trav.filter(f).asInstanceOf[IT[I]] + } + + override def oneToMany[I, O](trav: Collection[I])(f: I => Iterator[O]): CCOneToMany[O] = { + trav.flatMap(f).asInstanceOf[IT[O]] + } +} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala index cc1c15248..02e0c4c86 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala @@ -5,7 +5,8 @@ package object langv2 extends ExtensionClassImplicits with AnyTraversalImplicits implicit val singleOps = SingleOps implicit val optionOps = OptionOps - private val it2Ops = new IterableOnceOpsOps() - implicit def toIt2Ops[CC[_], C] = it2Ops.asInstanceOf[IterableOnceOpsOps[CC, C]] + implicit val iteratorOps = IteratorOps + private val it2Ops = new IterableOps() + implicit def toIt2Ops[IT[T] <: Iterable[T]] = it2Ops.asInstanceOf[IterableOps[IT]] } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala index b0c927b1e..47b1afbc3 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala @@ -9,33 +9,29 @@ import scala.jdk.CollectionConverters._ trait AstNodeTraversalImplicits { - implicit def toAstNodeTraversalSingle[I <: AstNode](trav: I): AstNodeTraversal[I, Single, Nothing] = { + implicit def toAstNodeTraversalSingle[I <: AstNode](trav: I): AstNodeTraversal[I, Single] = { new AstNodeTraversal(trav: Single[I]) } - implicit def toAstNodeTraversalGeneric[I <: AstNode](trav: Option[I]): AstNodeTraversal[I, Option, Nothing] = { + implicit def toAstNodeTraversalGeneric[I <: AstNode, IT[_]](trav: IT[I]): AstNodeTraversal[I, IT] = { new AstNodeTraversal(trav) } - implicit def toAstNodeTraversalIterOnceOps[I <: AstNode, CC[_], C](trav: IterableOnceOps[I, CC, C]) - : AstNodeTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { - new AstNodeTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) - } } -class AstNodeTraversal[I <: AstNode, IT[_], Marker](val trav: IT[I]) +class AstNodeTraversal[I <: AstNode, IT[_]](val trav: IT[I]) extends AnyVal { /** * Nodes of the AST rooted in this node, including the node itself. * */ @Doc("All nodes of the abstract syntax tree") - def ast(implicit ops1: TravOps[IT, Marker]) = + def ast(implicit ops1: TravOps[IT]) = trav.rFlatMap(_._astOut.asScala.asInstanceOf[Iterator[I]], _.emit) /** * Traverse only to AST nodes that are expressions * */ - def isExpression(implicit ops1: TravOps[IT, Marker]) = + def isExpression(implicit ops1: TravOps[IT]) = trav.collectAll[Expression] } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala index 17fcba413..9d1c7ead4 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala @@ -8,20 +8,16 @@ import scala.collection.IterableOnceOps import scala.jdk.CollectionConverters._ trait LocalTraversalImplicits { - implicit def toLocalTraversalSingle[I <: nodes.Local](trav: I): LocalTraversal[I, Single, Nothing] = { + implicit def toLocalTraversalSingle[I <: nodes.Local](trav: I): LocalTraversal[I, Single] = { new LocalTraversal(trav: Single[I]) } - implicit def toLocalTraversalGeneric[I <: nodes.Local](trav: Option[I]): LocalTraversal[I, Option, Nothing] = { + implicit def toLocalTraversalGeneric[I <: nodes.Local, IT[_]](trav: IT[I]): LocalTraversal[I, IT] = { new LocalTraversal(trav) } - implicit def toLocalTraversalIterOnceOps[I <: nodes.Local, CC[_], C](trav: IterableOnceOps[I, CC, C]) - : LocalTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { - new LocalTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) - } } -class LocalTraversal[I <: nodes.Local, IT[_], Marker](val trav: IT[I]) extends AnyVal { - def referencingIdentifiers(implicit ops1: TravOps[IT, Marker]): ops1.CCOneToMany[Identifier] = { +class LocalTraversal[I <: nodes.Local, IT[_]](val trav: IT[I]) extends AnyVal { + def referencingIdentifiers(implicit ops1: TravOps[IT]): ops1.CCOneToMany[Identifier] = { trav.flatMap(_._refIn.asScala.filter(_.label == NodeTypes.IDENTIFIER) .cast[Identifier]) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala index 845153403..956a6f793 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala @@ -9,29 +9,25 @@ import scala.collection.IterableOnceOps import scala.jdk.CollectionConverters._ trait MethodTraversalImplicits { - implicit def toMethodTraversalSingle[I <: nodes.Method](trav: I): MethodTraversal[I, Single, Nothing] = { + implicit def toMethodTraversalSingle[I <: nodes.Method](trav: I): MethodTraversal[I, Single] = { new MethodTraversal(trav: Single[I]) } - implicit def toMethodTraversalGeneric[I <: nodes.Method](trav: Option[I]): MethodTraversal[I, Option, Nothing] = { + implicit def toMethodTraversalGeneric[I <: nodes.Method, IT[_]](trav: IT[I]): MethodTraversal[I, IT] = { new MethodTraversal(trav) } - implicit def toMethodTraversalIterOnceOps[I <: nodes.Method, CC[_], C](trav: IterableOnceOps[I, CC, C]) - : MethodTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { - new MethodTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) - } } /** * A method, function, or procedure * */ @help.Traversal(elementType = classOf[nodes.Method]) -class MethodTraversal[I <: nodes.Method, IT[_], Marker](val trav: IT[I]) extends AnyVal { +class MethodTraversal[I <: nodes.Method, IT[_]](val trav: IT[I]) extends AnyVal { /** * Traverse to parameters of the method * */ @Doc("All parameters") - def parameter(implicit ops1: TravOps[IT, Marker]) = { + def parameter(implicit ops1: TravOps[IT]) = { ops1.oneToMany(trav)(_._astOut.asScala.collect { case par: nodes.MethodParameterIn => par }) } @@ -39,7 +35,7 @@ class MethodTraversal[I <: nodes.Method, IT[_], Marker](val trav: IT[I]) extends * Traverse to formal return parameter * */ @Doc("All formal return parameters") - def methodReturn(implicit ops1: TravOps[IT, Marker]) = { + def methodReturn(implicit ops1: TravOps[IT]) = { ops1.oneToOne(trav)(_._astOut.asScala.collectFirst { case ret: nodes.MethodReturn => ret }.get) } From be1e953c2dd5c81e596388123a60a9fde71c0ae5 Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Thu, 25 Nov 2021 13:29:59 +0100 Subject: [PATCH 24/27] Revert "Back to reality." This reverts commit d733696db6f9166f11da14a0a6c1730f4bf37eb6. --- performance/src/main/scala/MyTests.scala | 6 +- .../src/main/scala/some/SomeDomain.scala | 14 ++-- .../semanticcpg/langv2/AnyTraversal.scala | 25 ++++--- .../semanticcpg/langv2/TravOps.scala | 68 +++++++------------ .../semanticcpg/langv2/package.scala | 5 +- .../generalizations/AstNodeTraversal.scala | 14 ++-- .../types/structure/LocalTraversal.scala | 12 ++-- .../types/structure/MethodTraversal.scala | 14 ++-- 8 files changed, 79 insertions(+), 79 deletions(-) diff --git a/performance/src/main/scala/MyTests.scala b/performance/src/main/scala/MyTests.scala index 0b6fde73e..697138a07 100644 --- a/performance/src/main/scala/MyTests.scala +++ b/performance/src/main/scala/MyTests.scala @@ -11,7 +11,7 @@ import org.openjdk.jmh.annotations._ import MyTests._ import io.shiftleft.semanticcpg.language.types.structure.LocalReferencingIdentifiers -import scala.collection.{IndexedSeqView, View, mutable} +import scala.collection.{View, mutable} import scala.collection.mutable.{ArrayBuffer, ListBuffer} import org.openjdk.jmh.infra.Blackhole import overflowdb.traversal.Traversal @@ -131,8 +131,8 @@ class MyTestNew { def compileTest(state: MyState) = { val a: D2 = state.d1.toD2 val b: Option[D2] = Option(state.d1).toD2 - val c = toSynthOption(Array(state.d1).view.slice(1,2)).toD2 - val d: IndexedSeqView[D2] = c + val c = toSynthIter(Array(state.d1).view.slice(1,2)).toD2 + val d: View[D2] = c } } diff --git a/performance/src/main/scala/some/SomeDomain.scala b/performance/src/main/scala/some/SomeDomain.scala index 4493898f5..e01ca393e 100644 --- a/performance/src/main/scala/some/SomeDomain.scala +++ b/performance/src/main/scala/some/SomeDomain.scala @@ -7,19 +7,23 @@ import scala.collection.IterableOnceOps object SomeDomain { - implicit def toSynthSingle[I <: D1](p: I): SynthExt[I, Single] = { + implicit def toSynthSingle[I <: D1](p: I): SynthExt[I, Single, Nothing] = { new SynthExt(p: Single[I]) } - implicit def toSynthOption[I <: D1, IT[_]](trav: IT[I]): SynthExt[I, IT] = { + implicit def toSynthOption[I <: D1](trav: Option[I]): SynthExt[I, Option, Nothing] = { new SynthExt(trav) } - class SynthExt[I <: D1, IT[_]](val trav: IT[I]) extends AnyVal { - def toD2(implicit ops: TravOps[IT]) = { + implicit def toSynthIter[I <: D1, CC[_], C](trav: IterableOnceOps[I, CC, C]): SynthExt[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { + new SynthExt[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) + } + + class SynthExt[I <: D1, IT[_], Marker](val trav: IT[I]) extends AnyVal { + def toD2(implicit ops: TravOps[IT, Marker]) = { trav.map(_.x) } - def toD2Multi(implicit ops: TravOps[IT]) = { + def toD2Multi(implicit ops: TravOps[IT, Marker]) = { trav.flatMap(Iterator.single) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala index 138edb75c..ca162f4e0 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala @@ -5,49 +5,52 @@ import overflowdb.traversal.RepeatStepIterator import scala.collection.IterableOnceOps trait AnyTraversalImplicits { - implicit def toAnyTraversalInternal[I, IT[_]](trav: IT[I]) = { - new AnyTraversal[I, IT](trav) + implicit def toAnyTraversalIterOnceOps[I, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { + new AnyTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) + } + implicit def toAnyTraversalInternal[I, IT[_], Marker](trav: IT[I]) = { + new AnyTraversal[I, IT, Marker](trav) } } -class AnyTraversal[I, IT[_]](val trav: IT[I]) extends AnyVal { +class AnyTraversal[I, IT[_], Marker](val trav: IT[I]) extends AnyVal { - def cast[A](implicit ops: TravOps[IT]): ops.CCOneToOne[A] = { + def cast[A](implicit ops: TravOps[IT, Marker]): ops.CCOneToOne[A] = { trav.asInstanceOf[ops.CCOneToOne[A]] } - def map[O](f: I => O)(implicit ops: TravOps[IT]): ops.CCOneToOne[O] = { + def map[O](f: I => O)(implicit ops: TravOps[IT, Marker]): ops.CCOneToOne[O] = { ops.oneToOne(trav)(f) } - def filter(f: I => Boolean)(implicit ops: TravOps[IT]): ops.CCOneToBoolean[I] = { + def filter(f: I => Boolean)(implicit ops: TravOps[IT, Marker]): ops.CCOneToBoolean[I] = { ops.oneToBoolean(trav)(f) } - def flatMap[O](f: I => Iterator[O])(implicit ops: TravOps[IT]): ops.CCOneToMany[O] = { + def flatMap[O](f: I => Iterator[O])(implicit ops: TravOps[IT, Marker]): ops.CCOneToMany[O] = { ops.oneToMany(trav)(f) } // We dont use ClassTag and instead use our own IsInstanceOfOps for performance reasons. - def collectAll[T](implicit ops: TravOps[IT], isInstanceOfOps: IsInstanceOfOps[T]): ops.CCOneToBoolean[T] = { + def collectAll[T](implicit ops: TravOps[IT, Marker], isInstanceOfOps: IsInstanceOfOps[T]): ops.CCOneToBoolean[T] = { ops.oneToBoolean(trav)(isInstanceOfOps).asInstanceOf[ops.CCOneToBoolean[T]] } def rMap[O <: I](f: I => O, g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[IT]): ops.CCOneToMany[I] = { + implicit ops: TravOps[IT, Marker]): ops.CCOneToMany[I] = { val behaviour = g(new RepeatBehaviourBuilder()).build ops.oneToMany(trav)(i => new RepeatStepIterator(i, f.andThen(Iterator.single), behaviour)) } def rFlatMap[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[IT]): ops.CCOneToMany[I] = { + implicit ops: TravOps[IT, Marker]): ops.CCOneToMany[I] = { val behaviour = g(new RepeatBehaviourBuilder()).build ops.oneToMany(trav)(i => new RepeatStepIterator(i, f, behaviour)) } @deprecated("Use rFlatMap instead") def repeat[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[IT]): ops.CCOneToMany[I] = { + implicit ops: TravOps[IT, Marker]): ops.CCOneToMany[I] = { rFlatMap(f, g) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala index c33418fa9..6d3a94eec 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala @@ -10,7 +10,7 @@ import scala.collection.IterableOnceOps * @tparam FlatMapTraversal Flat map operation output traversal type. * Abbreviated in other generics as FT. */ -trait TravOps[_Collection[_]] { +trait TravOps[_Collection[_], ExtraTypes] { type Collection[T] = _Collection[T] type CCOneToOne[_] type CCOneToOption[_] @@ -23,21 +23,21 @@ trait TravOps[_Collection[_]] { def oneToMany[I, O](trav: Collection[I])(f: I => Iterator[O]): CCOneToMany[O] } -object SingleOps extends TravOps[Single] { +object SingleOps extends TravOps[Single, Nothing] { override type CCOneToOne[T] = T override type CCOneToOption[T] = Option[T] override type CCOneToBoolean[T] = Option[T] override type CCOneToMany[T] = Seq[T] - override def oneToOne[I, O](trav: Collection[I])(f: I => O): CCOneToOne[O] = { + override def oneToOne[I, O](trav: this.Collection[I])(f: I => O): CCOneToOne[O] = { f(trav) } - override def oneToOption[I, O](trav: Collection[I])(f: I => Option[O]): CCOneToOption[O] = { + override def oneToOption[I, O](trav: this.Collection[I])(f: I => Option[O]): CCOneToOption[O] = { f(trav) } - override def oneToBoolean[I](trav: Collection[I])(f: I => Boolean): CCOneToBoolean[I] = { + override def oneToBoolean[I](trav: this.Collection[I])(f: I => Boolean): CCOneToBoolean[I] = { if (f(trav)) { Some(trav) } else { @@ -45,30 +45,30 @@ object SingleOps extends TravOps[Single] { } } - override def oneToMany[I, O](trav: Collection[I])(f: I => Iterator[O]): CCOneToMany[O] = { + override def oneToMany[I, O](trav: this.Collection[I])(f: I => Iterator[O]): CCOneToMany[O] = { Seq.from(f(trav)) } } -object OptionOps extends TravOps[Option] { +object OptionOps extends TravOps[Option, Nothing] { override type CCOneToOne[T] = Option[T] override type CCOneToOption[T] = Option[T] override type CCOneToBoolean[T] = Option[T] override type CCOneToMany[T] = Seq[T] - override def oneToOne[I, O](trav: Collection[I])(f: I => O): CCOneToOne[O] = { + override def oneToOne[I, O](trav: this.Collection[I])(f: I => O): CCOneToOne[O] = { trav.map(f) } - override def oneToOption[I, O](trav: Collection[I])(f: I => Option[O]): CCOneToOption[O] = { + override def oneToOption[I, O](trav: this.Collection[I])(f: I => Option[O]): CCOneToOption[O] = { trav.flatMap(f) } - override def oneToBoolean[I](trav: Collection[I])(f: I => Boolean): CCOneToBoolean[I] = { + override def oneToBoolean[I](trav: this.Collection[I])(f: I => Boolean): CCOneToBoolean[I] = { trav.filter(f) } - override def oneToMany[I, O](trav: Collection[I])(f: I => Iterator[O]): CCOneToMany[O] = { + override def oneToMany[I, O](trav: this.Collection[I])(f: I => Iterator[O]): CCOneToMany[O] = { trav match { case Some(t) => Seq.from(f(t)) @@ -78,48 +78,30 @@ object OptionOps extends TravOps[Option] { } } -object IteratorOps extends TravOps[Iterator] { - override type CCOneToOne[T] = Iterator[T] - override type CCOneToOption[T] = Iterator[T] - override type CCOneToBoolean[T] = Iterator[T] - override type CCOneToMany[T] = Iterator[T] +class IterTypes[_CC[_], _C] { + type CC[T] = _CC[T] + type C = _C +} + +class IterableOnceOpsOps[CC[_], C] extends TravOps[({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] { + override type CCOneToOne[T] = CC[T] + override type CCOneToOption[T] = CC[T] + override type CCOneToBoolean[T] = C + override type CCOneToMany[T] = CC[T] - override def oneToOne[I, O](trav: Collection[I])(f: I => O): CCOneToOne[O] = { + override def oneToOne[I, O](trav: this.Collection[I])(f: I => O): CCOneToOne[O] = { trav.map(f) } - override def oneToOption[I, O](trav: Collection[I])(f: I => Option[O]): CCOneToOption[O] = { + override def oneToOption[I, O](trav: this.Collection[I])(f: I => Option[O]): CCOneToOption[O] = { trav.flatMap(f) } - override def oneToBoolean[I](trav: Collection[I])(f: I => Boolean): CCOneToBoolean[I] = { + override def oneToBoolean[I](trav: this.Collection[I])(f: I => Boolean): CCOneToBoolean[I] = { trav.filter(f) } - override def oneToMany[I, O](trav: Collection[I])(f: I => Iterator[O]): CCOneToMany[O] = { + override def oneToMany[I, O](trav: this.Collection[I])(f: I => Iterator[O]): CCOneToMany[O] = { trav.flatMap(f) } } - -class IterableOps[IT[T] <: Iterable[T]] extends TravOps[IT] { - override type CCOneToOne[T] = IT[T] - override type CCOneToOption[T] = IT[T] - override type CCOneToBoolean[T] = IT[T] - override type CCOneToMany[T] = IT[T] - - override def oneToOne[I, O](trav: Collection[I])(f: I => O): CCOneToOne[O] = { - trav.map(f).asInstanceOf[IT[O]] - } - - override def oneToOption[I, O](trav: Collection[I])(f: I => Option[O]): CCOneToOption[O] = { - trav.flatMap(f).asInstanceOf[IT[O]] - } - - override def oneToBoolean[I](trav: Collection[I])(f: I => Boolean): CCOneToBoolean[I] = { - trav.filter(f).asInstanceOf[IT[I]] - } - - override def oneToMany[I, O](trav: Collection[I])(f: I => Iterator[O]): CCOneToMany[O] = { - trav.flatMap(f).asInstanceOf[IT[O]] - } -} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala index 02e0c4c86..cc1c15248 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala @@ -5,8 +5,7 @@ package object langv2 extends ExtensionClassImplicits with AnyTraversalImplicits implicit val singleOps = SingleOps implicit val optionOps = OptionOps - implicit val iteratorOps = IteratorOps - private val it2Ops = new IterableOps() - implicit def toIt2Ops[IT[T] <: Iterable[T]] = it2Ops.asInstanceOf[IterableOps[IT]] + private val it2Ops = new IterableOnceOpsOps() + implicit def toIt2Ops[CC[_], C] = it2Ops.asInstanceOf[IterableOnceOpsOps[CC, C]] } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala index 47b1afbc3..b0c927b1e 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala @@ -9,29 +9,33 @@ import scala.jdk.CollectionConverters._ trait AstNodeTraversalImplicits { - implicit def toAstNodeTraversalSingle[I <: AstNode](trav: I): AstNodeTraversal[I, Single] = { + implicit def toAstNodeTraversalSingle[I <: AstNode](trav: I): AstNodeTraversal[I, Single, Nothing] = { new AstNodeTraversal(trav: Single[I]) } - implicit def toAstNodeTraversalGeneric[I <: AstNode, IT[_]](trav: IT[I]): AstNodeTraversal[I, IT] = { + implicit def toAstNodeTraversalGeneric[I <: AstNode](trav: Option[I]): AstNodeTraversal[I, Option, Nothing] = { new AstNodeTraversal(trav) } + implicit def toAstNodeTraversalIterOnceOps[I <: AstNode, CC[_], C](trav: IterableOnceOps[I, CC, C]) + : AstNodeTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { + new AstNodeTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) + } } -class AstNodeTraversal[I <: AstNode, IT[_]](val trav: IT[I]) +class AstNodeTraversal[I <: AstNode, IT[_], Marker](val trav: IT[I]) extends AnyVal { /** * Nodes of the AST rooted in this node, including the node itself. * */ @Doc("All nodes of the abstract syntax tree") - def ast(implicit ops1: TravOps[IT]) = + def ast(implicit ops1: TravOps[IT, Marker]) = trav.rFlatMap(_._astOut.asScala.asInstanceOf[Iterator[I]], _.emit) /** * Traverse only to AST nodes that are expressions * */ - def isExpression(implicit ops1: TravOps[IT]) = + def isExpression(implicit ops1: TravOps[IT, Marker]) = trav.collectAll[Expression] } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala index 9d1c7ead4..17fcba413 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala @@ -8,16 +8,20 @@ import scala.collection.IterableOnceOps import scala.jdk.CollectionConverters._ trait LocalTraversalImplicits { - implicit def toLocalTraversalSingle[I <: nodes.Local](trav: I): LocalTraversal[I, Single] = { + implicit def toLocalTraversalSingle[I <: nodes.Local](trav: I): LocalTraversal[I, Single, Nothing] = { new LocalTraversal(trav: Single[I]) } - implicit def toLocalTraversalGeneric[I <: nodes.Local, IT[_]](trav: IT[I]): LocalTraversal[I, IT] = { + implicit def toLocalTraversalGeneric[I <: nodes.Local](trav: Option[I]): LocalTraversal[I, Option, Nothing] = { new LocalTraversal(trav) } + implicit def toLocalTraversalIterOnceOps[I <: nodes.Local, CC[_], C](trav: IterableOnceOps[I, CC, C]) + : LocalTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { + new LocalTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) + } } -class LocalTraversal[I <: nodes.Local, IT[_]](val trav: IT[I]) extends AnyVal { - def referencingIdentifiers(implicit ops1: TravOps[IT]): ops1.CCOneToMany[Identifier] = { +class LocalTraversal[I <: nodes.Local, IT[_], Marker](val trav: IT[I]) extends AnyVal { + def referencingIdentifiers(implicit ops1: TravOps[IT, Marker]): ops1.CCOneToMany[Identifier] = { trav.flatMap(_._refIn.asScala.filter(_.label == NodeTypes.IDENTIFIER) .cast[Identifier]) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala index 956a6f793..845153403 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala @@ -9,25 +9,29 @@ import scala.collection.IterableOnceOps import scala.jdk.CollectionConverters._ trait MethodTraversalImplicits { - implicit def toMethodTraversalSingle[I <: nodes.Method](trav: I): MethodTraversal[I, Single] = { + implicit def toMethodTraversalSingle[I <: nodes.Method](trav: I): MethodTraversal[I, Single, Nothing] = { new MethodTraversal(trav: Single[I]) } - implicit def toMethodTraversalGeneric[I <: nodes.Method, IT[_]](trav: IT[I]): MethodTraversal[I, IT] = { + implicit def toMethodTraversalGeneric[I <: nodes.Method](trav: Option[I]): MethodTraversal[I, Option, Nothing] = { new MethodTraversal(trav) } + implicit def toMethodTraversalIterOnceOps[I <: nodes.Method, CC[_], C](trav: IterableOnceOps[I, CC, C]) + : MethodTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { + new MethodTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) + } } /** * A method, function, or procedure * */ @help.Traversal(elementType = classOf[nodes.Method]) -class MethodTraversal[I <: nodes.Method, IT[_]](val trav: IT[I]) extends AnyVal { +class MethodTraversal[I <: nodes.Method, IT[_], Marker](val trav: IT[I]) extends AnyVal { /** * Traverse to parameters of the method * */ @Doc("All parameters") - def parameter(implicit ops1: TravOps[IT]) = { + def parameter(implicit ops1: TravOps[IT, Marker]) = { ops1.oneToMany(trav)(_._astOut.asScala.collect { case par: nodes.MethodParameterIn => par }) } @@ -35,7 +39,7 @@ class MethodTraversal[I <: nodes.Method, IT[_]](val trav: IT[I]) extends AnyVal * Traverse to formal return parameter * */ @Doc("All formal return parameters") - def methodReturn(implicit ops1: TravOps[IT]) = { + def methodReturn(implicit ops1: TravOps[IT, Marker]) = { ops1.oneToOne(trav)(_._astOut.asScala.collectFirst { case ret: nodes.MethodReturn => ret }.get) } From 8595014c02b26635561a24a8381be6d6edfb9bb5 Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Thu, 25 Nov 2021 15:23:44 +0100 Subject: [PATCH 25/27] Individual applier traits. Compiles. --- .../src/main/scala/some/SomeDomain.scala | 10 +- .../generalizations/AstNodeTraversal.scala | 1 - .../semanticcpg/langv2/AnyTraversal.scala | 32 +--- .../semanticcpg/langv2/TravOps.scala | 180 ++++++++++-------- .../semanticcpg/langv2/package.scala | 23 ++- .../generalizations/AstNodeTraversal.scala | 22 +-- .../types/structure/LocalTraversal.scala | 18 +- .../types/structure/MethodTraversal.scala | 22 +-- 8 files changed, 161 insertions(+), 147 deletions(-) diff --git a/performance/src/main/scala/some/SomeDomain.scala b/performance/src/main/scala/some/SomeDomain.scala index e01ca393e..242cbe174 100644 --- a/performance/src/main/scala/some/SomeDomain.scala +++ b/performance/src/main/scala/some/SomeDomain.scala @@ -19,12 +19,12 @@ object SomeDomain { new SynthExt[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) } - class SynthExt[I <: D1, IT[_], Marker](val trav: IT[I]) extends AnyVal { - def toD2(implicit ops: TravOps[IT, Marker]) = { - trav.map(_.x) + class SynthExt[I <: D1, IT[_], Marker](val in: IT[I]) extends AnyVal { + def toD2(implicit applier: ToOne[IT, Marker]) = { + applier.apply(in)(_.x) } - def toD2Multi(implicit ops: TravOps[IT, Marker]) = { - trav.flatMap(Iterator.single) + def toD2Multi(implicit applier: ToMany[IT, Marker]) = { + applier.apply(in)(Iterator.single) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala index 29cd47f20..26107f2d7 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/expressions/generalizations/AstNodeTraversal.scala @@ -3,7 +3,6 @@ package io.shiftleft.semanticcpg.language.types.expressions.generalizations import io.shiftleft.codepropertygraph.generated.nodes._ import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, nodes} import io.shiftleft.semanticcpg.language._ -import io.shiftleft.semanticcpg.langv2._ import io.shiftleft.semanticcpg.langv3.{Helper, Kernel1ToN, Kernel1ToO} import overflowdb.traversal.help.Doc import overflowdb.traversal.{Traversal, help} diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala index ca162f4e0..cf88709fd 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala @@ -5,7 +5,7 @@ import overflowdb.traversal.RepeatStepIterator import scala.collection.IterableOnceOps trait AnyTraversalImplicits { - implicit def toAnyTraversalIterOnceOps[I, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { + implicit def toAnyTraversalIter[I, CC[_], C](trav: IterableOnceOps[I, CC, C]) = { new AnyTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) } implicit def toAnyTraversalInternal[I, IT[_], Marker](trav: IT[I]) = { @@ -15,42 +15,30 @@ trait AnyTraversalImplicits { class AnyTraversal[I, IT[_], Marker](val trav: IT[I]) extends AnyVal { - def cast[A](implicit ops: TravOps[IT, Marker]): ops.CCOneToOne[A] = { - trav.asInstanceOf[ops.CCOneToOne[A]] - } - - def map[O](f: I => O)(implicit ops: TravOps[IT, Marker]): ops.CCOneToOne[O] = { - ops.oneToOne(trav)(f) - } - - def filter(f: I => Boolean)(implicit ops: TravOps[IT, Marker]): ops.CCOneToBoolean[I] = { - ops.oneToBoolean(trav)(f) - } - - def flatMap[O](f: I => Iterator[O])(implicit ops: TravOps[IT, Marker]): ops.CCOneToMany[O] = { - ops.oneToMany(trav)(f) + def cast[A](implicit applier: ToOne[IT, Marker]): applier.OUT[A] = { + trav.asInstanceOf[applier.OUT[A]] } // We dont use ClassTag and instead use our own IsInstanceOfOps for performance reasons. - def collectAll[T](implicit ops: TravOps[IT, Marker], isInstanceOfOps: IsInstanceOfOps[T]): ops.CCOneToBoolean[T] = { - ops.oneToBoolean(trav)(isInstanceOfOps).asInstanceOf[ops.CCOneToBoolean[T]] + def collectAll[T](implicit applier: ToBoolean[IT, Marker], isInstanceOfOps: IsInstanceOfOps[T]): applier.OUT[T] = { + applier.apply(trav)(isInstanceOfOps).asInstanceOf[applier.OUT[T]] } def rMap[O <: I](f: I => O, g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[IT, Marker]): ops.CCOneToMany[I] = { + implicit applier: ToMany[IT, Marker]): applier.OUT[I] = { val behaviour = g(new RepeatBehaviourBuilder()).build - ops.oneToMany(trav)(i => new RepeatStepIterator(i, f.andThen(Iterator.single), behaviour)) + applier.apply(trav)(i => new RepeatStepIterator(i, f.andThen(Iterator.single), behaviour)) } def rFlatMap[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[IT, Marker]): ops.CCOneToMany[I] = { + implicit applier: ToMany[IT, Marker]): applier.OUT[I] = { val behaviour = g(new RepeatBehaviourBuilder()).build - ops.oneToMany(trav)(i => new RepeatStepIterator(i, f, behaviour)) + applier.apply(trav)(i => new RepeatStepIterator(i, f, behaviour)) } @deprecated("Use rFlatMap instead") def repeat[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( - implicit ops: TravOps[IT, Marker]): ops.CCOneToMany[I] = { + implicit applier: ToMany[IT, Marker]): applier.OUT[I] = { rFlatMap(f, g) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala index 6d3a94eec..8fee0ffa3 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala @@ -2,106 +2,118 @@ package io.shiftleft.semanticcpg.langv2 import scala.collection.IterableOnceOps -/** Traversal operations trait. - * Implementations of this trait handle the basic traversal operations for the - * different kind of input traversal types. - * - * @tparam InputTraversal Input traversal type. Abbreviated in other generics as IT. - * @tparam FlatMapTraversal Flat map operation output traversal type. - * Abbreviated in other generics as FT. - */ -trait TravOps[_Collection[_], ExtraTypes] { - type Collection[T] = _Collection[T] - type CCOneToOne[_] - type CCOneToOption[_] - type CCOneToBoolean[_] - type CCOneToMany[_] - - def oneToOne[I, O](trav: Collection[I])(f: I => O): CCOneToOne[O] - def oneToOption[I, O](trav: Collection[I])(f: I => Option[O]): CCOneToOption[O] - def oneToBoolean[I](trav: Collection[I])(f: I => Boolean): CCOneToBoolean[I] - def oneToMany[I, O](trav: Collection[I])(f: I => Iterator[O]): CCOneToMany[O] -} - -object SingleOps extends TravOps[Single, Nothing] { - override type CCOneToOne[T] = T - override type CCOneToOption[T] = Option[T] - override type CCOneToBoolean[T] = Option[T] - override type CCOneToMany[T] = Seq[T] - - override def oneToOne[I, O](trav: this.Collection[I])(f: I => O): CCOneToOne[O] = { - f(trav) - } - - override def oneToOption[I, O](trav: this.Collection[I])(f: I => Option[O]): CCOneToOption[O] = { - f(trav) - } - - override def oneToBoolean[I](trav: this.Collection[I])(f: I => Boolean): CCOneToBoolean[I] = { - if (f(trav)) { - Some(trav) +class IterTypes[_CC[_], _C] { + type CC[T] = _CC[T] + type C = _C +} + +trait ToOne[_IN[_], ExtraTypes] { + type IN[T] = _IN[T] + type OUT[_] + + def apply[I, O](in: IN[I])(f: I => O): OUT[O] +} + +object SingleToOne extends ToOne[Single, Nothing] { + override type OUT[T] = T + + override def apply[I, O](in: IN[I])(f: I => O): OUT[O] = f(in) +} + +object OptionToOne extends ToOne[Option, Nothing] { + override type OUT[T] = Option[T] + + override def apply[I, O](in: IN[I])(f: I => O): OUT[O] = in.map(f) +} + +class IterToOne[CC[_], C] extends ToOne[({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] { + override type OUT[T] = CC[T] + + override def apply[I, O](in: IN[I])(f: I => O): OUT[O] = in.map(f) +} + +trait ToBoolean[_IN[_], ExtraTypes] { + type IN[T] = _IN[T] + type OUT[_] + + def apply[I, O](in: IN[I])(f: I => Boolean): OUT[I] +} + +object SingleToBoolean extends ToBoolean[Single, Nothing] { + override type OUT[T] = Option[T] + + override def apply[I, O](in: IN[I])(f: I => Boolean): OUT[I] = + if (f(in)) { + Some(in) } else { None } - } +} + +object OptionToBoolean extends ToBoolean[Option, Nothing] { + override type OUT[T] = Option[T] - override def oneToMany[I, O](trav: this.Collection[I])(f: I => Iterator[O]): CCOneToMany[O] = { - Seq.from(f(trav)) - } + override def apply[I, O](in: IN[I])(f: I => Boolean): OUT[I] = in.filter(f) } -object OptionOps extends TravOps[Option, Nothing] { - override type CCOneToOne[T] = Option[T] - override type CCOneToOption[T] = Option[T] - override type CCOneToBoolean[T] = Option[T] - override type CCOneToMany[T] = Seq[T] +class IterToBoolean[CC[_], C] extends ToBoolean[({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] { + override type OUT[T] = C - override def oneToOne[I, O](trav: this.Collection[I])(f: I => O): CCOneToOne[O] = { - trav.map(f) - } + override def apply[I, O](in: IN[I])(f: I => Boolean): OUT[I] = in.filter(f) +} - override def oneToOption[I, O](trav: this.Collection[I])(f: I => Option[O]): CCOneToOption[O] = { - trav.flatMap(f) - } +trait ToOption[_IN[_], ExtraTypes] { + type IN[T] = _IN[T] + type OUT[_] - override def oneToBoolean[I](trav: this.Collection[I])(f: I => Boolean): CCOneToBoolean[I] = { - trav.filter(f) - } + def apply[I, O](in: IN[I])(f: I => Option[O]): OUT[O] +} - override def oneToMany[I, O](trav: this.Collection[I])(f: I => Iterator[O]): CCOneToMany[O] = { - trav match { - case Some(t) => - Seq.from(f(t)) - case None => - Seq.empty - } - } +object SingleToOption extends ToOption[Single, Nothing] { + override type OUT[T] = Option[T] + + override def apply[I, O](in: IN[I])(f: I => Option[O]): OUT[O] = f(in) } -class IterTypes[_CC[_], _C] { - type CC[T] = _CC[T] - type C = _C +object OptionToOption extends ToOption[Option, Nothing] { + override type OUT[T] = Option[T] + + override def apply[I, O](in: IN[I])(f: I => Option[O]): OUT[O] = in.flatMap(f) +} + +class IterToOption[CC[_], C] extends ToOption[({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] { + override type OUT[T] = CC[T] + + override def apply[I, O](in: IN[I])(f: I => Option[O]): OUT[O] = in.flatMap(f) +} + +trait ToMany[_IN[_], ExtraTypes] { + type IN[T] = _IN[T] + type OUT[_] + + def apply[I, O](in: IN[I])(f: I => IterableOnce[O]): OUT[O] } -class IterableOnceOpsOps[CC[_], C] extends TravOps[({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] { - override type CCOneToOne[T] = CC[T] - override type CCOneToOption[T] = CC[T] - override type CCOneToBoolean[T] = C - override type CCOneToMany[T] = CC[T] +object SingleToMany extends ToMany[Single, Nothing] { + override type OUT[T] = Seq[T] - override def oneToOne[I, O](trav: this.Collection[I])(f: I => O): CCOneToOne[O] = { - trav.map(f) - } + override def apply[I, O](in: IN[I])(f: I => IterableOnce[O]): OUT[O] = Seq.from(f(in)) +} - override def oneToOption[I, O](trav: this.Collection[I])(f: I => Option[O]): CCOneToOption[O] = { - trav.flatMap(f) - } +object OptionToMany extends ToMany[Option, Nothing] { + override type OUT[T] = Seq[T] + + override def apply[I, O](in: IN[I])(f: I => IterableOnce[O]): OUT[O] = + in match { + case Some(i) => + Seq.from(f(i)) + case None => + Seq.empty + } +} - override def oneToBoolean[I](trav: this.Collection[I])(f: I => Boolean): CCOneToBoolean[I] = { - trav.filter(f) - } +class IterToMany[CC[_], C] extends ToMany[({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] { + override type OUT[T] = CC[T] - override def oneToMany[I, O](trav: this.Collection[I])(f: I => Iterator[O]): CCOneToMany[O] = { - trav.flatMap(f) - } + override def apply[I, O](in: IN[I])(f: I => IterableOnce[O]): OUT[O] = in.flatMap(f) } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala index cc1c15248..627ae790d 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala @@ -3,9 +3,24 @@ package io.shiftleft.semanticcpg package object langv2 extends ExtensionClassImplicits with AnyTraversalImplicits with InstanceOfOpsImplicits { type Single[T] = T - implicit val singleOps = SingleOps - implicit val optionOps = OptionOps - private val it2Ops = new IterableOnceOpsOps() - implicit def toIt2Ops[CC[_], C] = it2Ops.asInstanceOf[IterableOnceOpsOps[CC, C]] + implicit val singleToOne: SingleToOne.type = SingleToOne + implicit val optionToOne: OptionToOne.type = OptionToOne + private val _iterToOne = new IterToOne() + implicit def iterToOne[CC[_], C]: IterToOne[CC, C] = _iterToOne.asInstanceOf[IterToOne[CC, C]] + + implicit val singleToBoolean: SingleToBoolean.type = SingleToBoolean + implicit val optionToBoolean: OptionToBoolean.type = OptionToBoolean + private val _iterToBoolean = new IterToBoolean() + implicit def iterToBoolean[CC[_], C]: IterToBoolean[CC, C] = _iterToBoolean.asInstanceOf[IterToBoolean[CC, C]] + + implicit val singleToOption: SingleToOption.type = SingleToOption + implicit val optionToOption: OptionToOption.type = OptionToOption + private val _iterToOption = new IterToOption() + implicit def iterToOption[CC[_], C]: IterToOption[CC, C] = _iterToOption.asInstanceOf[IterToOption[CC, C]] + + implicit val singleToMany: SingleToMany.type = SingleToMany + implicit val optionToMany: OptionToMany.type = OptionToMany + private val _iterToMany = new IterToMany() + implicit def iterToMany[CC[_], C]: IterToMany[CC, C] = _iterToMany.asInstanceOf[IterToMany[CC, C]] } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala index b0c927b1e..d4bb45e62 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala @@ -9,34 +9,34 @@ import scala.jdk.CollectionConverters._ trait AstNodeTraversalImplicits { - implicit def toAstNodeTraversalSingle[I <: AstNode](trav: I): AstNodeTraversal[I, Single, Nothing] = { - new AstNodeTraversal(trav: Single[I]) + implicit def toAstNodeTraversalSingle[I <: AstNode](in: I): AstNodeTraversal[I, Single, Nothing] = { + new AstNodeTraversal(in: Single[I]) } - implicit def toAstNodeTraversalGeneric[I <: AstNode](trav: Option[I]): AstNodeTraversal[I, Option, Nothing] = { - new AstNodeTraversal(trav) + implicit def toAstNodeTraversalOption[I <: AstNode](in: Option[I]): AstNodeTraversal[I, Option, Nothing] = { + new AstNodeTraversal(in) } - implicit def toAstNodeTraversalIterOnceOps[I <: AstNode, CC[_], C](trav: IterableOnceOps[I, CC, C]) + implicit def toAstNodeTraversalIter[I <: AstNode, CC[_], C](in: IterableOnceOps[I, CC, C]) : AstNodeTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { - new AstNodeTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) + new AstNodeTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](in) } } -class AstNodeTraversal[I <: AstNode, IT[_], Marker](val trav: IT[I]) +class AstNodeTraversal[I <: AstNode, IT[_], Marker](val in: IT[I]) extends AnyVal { /** * Nodes of the AST rooted in this node, including the node itself. * */ @Doc("All nodes of the abstract syntax tree") - def ast(implicit ops1: TravOps[IT, Marker]) = - trav.rFlatMap(_._astOut.asScala.asInstanceOf[Iterator[I]], _.emit) + def ast(implicit applier: ToMany[IT, Marker]) = + in.rFlatMap(_._astOut.asScala.asInstanceOf[Iterator[I]], _.emit) /** * Traverse only to AST nodes that are expressions * */ - def isExpression(implicit ops1: TravOps[IT, Marker]) = - trav.collectAll[Expression] + def isExpression(implicit applier: ToBoolean[IT, Marker]) = + in.collectAll[Expression] } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala index 17fcba413..527d1f331 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/LocalTraversal.scala @@ -8,21 +8,21 @@ import scala.collection.IterableOnceOps import scala.jdk.CollectionConverters._ trait LocalTraversalImplicits { - implicit def toLocalTraversalSingle[I <: nodes.Local](trav: I): LocalTraversal[I, Single, Nothing] = { - new LocalTraversal(trav: Single[I]) + implicit def toLocalTraversalSingle[I <: nodes.Local](in: I): LocalTraversal[I, Single, Nothing] = { + new LocalTraversal(in: Single[I]) } - implicit def toLocalTraversalGeneric[I <: nodes.Local](trav: Option[I]): LocalTraversal[I, Option, Nothing] = { - new LocalTraversal(trav) + implicit def toLocalTraversalOption[I <: nodes.Local](in: Option[I]): LocalTraversal[I, Option, Nothing] = { + new LocalTraversal(in) } - implicit def toLocalTraversalIterOnceOps[I <: nodes.Local, CC[_], C](trav: IterableOnceOps[I, CC, C]) + implicit def toLocalTraversalIter[I <: nodes.Local, CC[_], C](in: IterableOnceOps[I, CC, C]) : LocalTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { - new LocalTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) + new LocalTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](in) } } -class LocalTraversal[I <: nodes.Local, IT[_], Marker](val trav: IT[I]) extends AnyVal { - def referencingIdentifiers(implicit ops1: TravOps[IT, Marker]): ops1.CCOneToMany[Identifier] = { - trav.flatMap(_._refIn.asScala.filter(_.label == NodeTypes.IDENTIFIER) +class LocalTraversal[I <: nodes.Local, IT[_], Marker](val in: IT[I]) extends AnyVal { + def referencingIdentifiers(implicit applier: ToMany[IT, Marker]): applier.OUT[Identifier] = { + applier.apply(in)(_._refIn.asScala.filter(_.label == NodeTypes.IDENTIFIER) .cast[Identifier]) } } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala index 845153403..df67d406e 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala @@ -9,15 +9,15 @@ import scala.collection.IterableOnceOps import scala.jdk.CollectionConverters._ trait MethodTraversalImplicits { - implicit def toMethodTraversalSingle[I <: nodes.Method](trav: I): MethodTraversal[I, Single, Nothing] = { - new MethodTraversal(trav: Single[I]) + implicit def toMethodTraversalSingle[I <: nodes.Method](in: I): MethodTraversal[I, Single, Nothing] = { + new MethodTraversal(in: Single[I]) } - implicit def toMethodTraversalGeneric[I <: nodes.Method](trav: Option[I]): MethodTraversal[I, Option, Nothing] = { - new MethodTraversal(trav) + implicit def toMethodTraversalOption[I <: nodes.Method](in: Option[I]): MethodTraversal[I, Option, Nothing] = { + new MethodTraversal(in) } - implicit def toMethodTraversalIterOnceOps[I <: nodes.Method, CC[_], C](trav: IterableOnceOps[I, CC, C]) + implicit def toMethodTraversalIter[I <: nodes.Method, CC[_], C](in: IterableOnceOps[I, CC, C]) : MethodTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { - new MethodTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) + new MethodTraversal[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](in) } } @@ -25,22 +25,22 @@ trait MethodTraversalImplicits { * A method, function, or procedure * */ @help.Traversal(elementType = classOf[nodes.Method]) -class MethodTraversal[I <: nodes.Method, IT[_], Marker](val trav: IT[I]) extends AnyVal { +class MethodTraversal[I <: nodes.Method, IT[_], Marker](val in: IT[I]) extends AnyVal { /** * Traverse to parameters of the method * */ @Doc("All parameters") - def parameter(implicit ops1: TravOps[IT, Marker]) = { - ops1.oneToMany(trav)(_._astOut.asScala.collect { case par: nodes.MethodParameterIn => par }) + def parameter(implicit applier: ToMany[IT, Marker]) = { + applier.apply(in)(_._astOut.asScala.collect { case par: nodes.MethodParameterIn => par }) } /** * Traverse to formal return parameter * */ @Doc("All formal return parameters") - def methodReturn(implicit ops1: TravOps[IT, Marker]) = { - ops1.oneToOne(trav)(_._astOut.asScala.collectFirst { case ret: nodes.MethodReturn => ret }.get) + def methodReturn(implicit applier: ToOne[IT, Marker]) = { + applier.apply(in)(_._astOut.asScala.collectFirst { case ret: nodes.MethodReturn => ret }.get) } } From 41837f6160c19b6f6c56722060c0d823c854574c Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Mon, 29 Nov 2021 10:26:34 +0100 Subject: [PATCH 26/27] Added ToGlobal. --- performance/src/main/scala/MyTests.scala | 9 ++- .../src/main/scala/some/SomeDomain.scala | 6 +- .../semanticcpg/langv2/AnyTraversal.scala | 2 +- .../semanticcpg/langv2/TravOps.scala | 63 ++++++++++++++++++- .../semanticcpg/langv2/package.scala | 7 +++ .../generalizations/AstNodeTraversal.scala | 5 +- .../types/structure/MethodTraversal.scala | 4 ++ 7 files changed, 88 insertions(+), 8 deletions(-) diff --git a/performance/src/main/scala/MyTests.scala b/performance/src/main/scala/MyTests.scala index 697138a07..8794f3f1f 100644 --- a/performance/src/main/scala/MyTests.scala +++ b/performance/src/main/scala/MyTests.scala @@ -11,7 +11,7 @@ import org.openjdk.jmh.annotations._ import MyTests._ import io.shiftleft.semanticcpg.language.types.structure.LocalReferencingIdentifiers -import scala.collection.{View, mutable} +import scala.collection.{IterableOnceOps, SeqOps, View, mutable} import scala.collection.mutable.{ArrayBuffer, ListBuffer} import org.openjdk.jmh.infra.Blackhole import overflowdb.traversal.Traversal @@ -133,6 +133,13 @@ class MyTestNew { val b: Option[D2] = Option(state.d1).toD2 val c = toSynthIter(Array(state.d1).view.slice(1,2)).toD2 val d: View[D2] = c + val z: scala.collection.Seq[Int] = ArrayBuffer.empty[Int] + + Iterable.single(state.d1).toD2Multi + Iterator.single(state.d1).toD2Multi + + val a1 = Iterable.single(state.d1).doGlobal + val a2 = Iterator.single(state.d1).doGlobal } } diff --git a/performance/src/main/scala/some/SomeDomain.scala b/performance/src/main/scala/some/SomeDomain.scala index 242cbe174..d94268306 100644 --- a/performance/src/main/scala/some/SomeDomain.scala +++ b/performance/src/main/scala/some/SomeDomain.scala @@ -15,7 +15,8 @@ object SomeDomain { new SynthExt(trav) } - implicit def toSynthIter[I <: D1, CC[_], C](trav: IterableOnceOps[I, CC, C]): SynthExt[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { + implicit def toSynthIter[I <: D1, CC[_], C](trav: IterableOnceOps[I, CC, C]) + : SynthExt[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] = { new SynthExt[I, ({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]](trav) } @@ -26,6 +27,9 @@ object SomeDomain { def toD2Multi(implicit applier: ToMany[IT, Marker]) = { applier.apply(in)(Iterator.single) } + def doGlobal(implicit applier: ToGlobal[IT, Marker]) = { + applier.apply(in)(x => x) + } } case class D1(x: D2) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala index cf88709fd..06e6edd17 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/AnyTraversal.scala @@ -36,7 +36,7 @@ class AnyTraversal[I, IT[_], Marker](val trav: IT[I]) extends AnyVal { applier.apply(trav)(i => new RepeatStepIterator(i, f, behaviour)) } - @deprecated("Use rFlatMap instead") + //@deprecated("Use rFlatMap instead") def repeat[O <: I](f: I => Iterator[O], g: RepeatBehaviourBuilder[I] => RepeatBehaviourBuilder[I] = identity)( implicit applier: ToMany[IT, Marker]): applier.OUT[I] = { rFlatMap(f, g) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala index 8fee0ffa3..650664218 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala @@ -1,8 +1,9 @@ package io.shiftleft.semanticcpg.langv2 import scala.collection.IterableOnceOps +import scala.collection.mutable.ArrayBuffer -class IterTypes[_CC[_], _C] { +class IterTypes[_CC[T], _C] { type CC[T] = _CC[T] type C = _C } @@ -112,8 +113,64 @@ object OptionToMany extends ToMany[Option, Nothing] { } } -class IterToMany[CC[_], C] extends ToMany[({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] { +class IterToMany[CC[_], C] + extends ToMany[({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] { override type OUT[T] = CC[T] - override def apply[I, O](in: IN[I])(f: I => IterableOnce[O]): OUT[O] = in.flatMap(f) + override def apply[I, O](in: IN[I])(f: I => IterableOnce[O]): OUT[O] = { + in.flatMap(f) + } +} + +trait ToGlobal[_IN[_], ExtraTypes] { + type IN[T] = _IN[T] + type OUT[_] + + def apply[I, O](in: IN[I])(f: collection.Seq[I] => collection.Seq[O]): OUT[O] +} + +object SingleToGlobal extends ToGlobal[Single, Nothing] { + override type OUT[T] = collection.Seq[T] + + override def apply[I, O](in: IN[I])(f: collection.Seq[I] => collection.Seq[O]): OUT[O] = { + f(in::Nil) + } +} + +object OptionToGlobal extends ToGlobal[Option, Nothing] { + override type OUT[T] = collection.Seq[T] + + override def apply[I, O](in: IN[I])(f: collection.Seq[I] => collection.Seq[O]): OUT[O] = { + in match { + case Some(i) => + f(i::Nil) + case None => + f(Nil) + } + } +} + +class IterToGlobal[CC[_], C] extends ToGlobal[({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] { + override type OUT[T] = CC[T] + + override def apply[I, O](in: IN[I])(f: collection.Seq[I] => collection.Seq[O]): OUT[O] = { + in match { + case iterable: Iterable[I] => + iterable.iterableFactory.from(f(iterable.toSeq)).asInstanceOf[CC[O]] + case iterator: Iterator[I] => + new Iterator[O] { + private var resultIt: Iterator[O] = _ + override def hasNext: Boolean = { + if (resultIt == null) { + resultIt = f(in.to(ArrayBuffer)).iterator + } + resultIt.hasNext + } + override def next(): O = { + resultIt.next() + } + }.asInstanceOf[CC[O]] + } + } + } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala index 627ae790d..6c95d184d 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/package.scala @@ -1,5 +1,7 @@ package io.shiftleft.semanticcpg +import scala.collection.IterableOnceOps + package object langv2 extends ExtensionClassImplicits with AnyTraversalImplicits with InstanceOfOpsImplicits { type Single[T] = T @@ -23,4 +25,9 @@ package object langv2 extends ExtensionClassImplicits with AnyTraversalImplicits private val _iterToMany = new IterToMany() implicit def iterToMany[CC[_], C]: IterToMany[CC, C] = _iterToMany.asInstanceOf[IterToMany[CC, C]] + implicit val singleToGlobal: SingleToGlobal.type = SingleToGlobal + implicit val optionToGlobal: OptionToGlobal.type = OptionToGlobal + private val _iterToGlobal = new IterToGlobal() + implicit def iterToGlobal[CC[_], C]: IterToGlobal[CC, C] = _iterToGlobal.asInstanceOf[IterToGlobal[CC, C]] + } diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala index d4bb45e62..be809e768 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/expressions/generalizations/AstNodeTraversal.scala @@ -29,8 +29,9 @@ class AstNodeTraversal[I <: AstNode, IT[_], Marker](val in: IT[I]) * Nodes of the AST rooted in this node, including the node itself. * */ @Doc("All nodes of the abstract syntax tree") - def ast(implicit applier: ToMany[IT, Marker]) = - in.rFlatMap(_._astOut.asScala.asInstanceOf[Iterator[I]], _.emit) + def ast(implicit applier: ToMany[IT, Marker]) = { + in.repeat(_._astOut.asScala.asInstanceOf[Iterator[I]], _.emit) + } /** * Traverse only to AST nodes that are expressions diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala index df67d406e..946702cc5 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/types/structure/MethodTraversal.scala @@ -43,4 +43,8 @@ class MethodTraversal[I <: nodes.Method, IT[_], Marker](val in: IT[I]) extends A applier.apply(in)(_._astOut.asScala.collectFirst { case ret: nodes.MethodReturn => ret }.get) } + //def toParameterCount(implicit applier: ToGlobal[IT, Marker]) = { + // applier.apply(in)(_.map(_.parameter).size) + //} + } From 2c2de37bc7006b1ec69dc7d0c10f79bbfb87540f Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Mon, 29 Nov 2021 16:46:12 +0100 Subject: [PATCH 27/27] Add ToGlobalEnd trait. --- .../types/structure/LocalTraversal.scala | 1 - .../semanticcpg/langv2/TravOps.scala | 29 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala index 488c17d07..c9333c66b 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/types/structure/LocalTraversal.scala @@ -5,7 +5,6 @@ import io.shiftleft.codepropertygraph.generated.nodes import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} import io.shiftleft.semanticcpg.language.MySteps._ import io.shiftleft.semanticcpg.language.EPipe -import io.shiftleft.semanticcpg.langv2._ import io.shiftleft.semanticcpg.langv3.{Helper, Kernel1ToN, Kernel1ToO} import overflowdb.traversal.Traversal diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala index 650664218..4e2a918ed 100644 --- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala +++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/langv2/TravOps.scala @@ -2,6 +2,7 @@ package io.shiftleft.semanticcpg.langv2 import scala.collection.IterableOnceOps import scala.collection.mutable.ArrayBuffer +import scala.collection.Seq class IterTypes[_CC[T], _C] { type CC[T] = _CC[T] @@ -172,5 +173,33 @@ class IterToGlobal[CC[_], C] extends ToGlobal[({type X[A] = IterableOnceOps[A, C }.asInstanceOf[CC[O]] } } +} + +trait ToGlobalEnd[_IN[_], ExtraTypes] { + type IN[T] = _IN[T] + + def apply[I, O](in: IN[I])(f: Seq[I] => O): O +} +object SingleToGlobalEnd extends ToGlobalEnd[Single, Nothing] { + override def apply[I, O](in: IN[I])(f: Seq[I] => O): O = { + f(in::Nil) + } +} + +object OptionToGlobalEnd extends ToGlobalEnd[Option, Nothing] { + override def apply[I, O](in: IN[I])(f: Seq[I] => O): O = { + in match { + case Some(i) => + f(i::Nil) + case None => + f(Nil) + } + } +} + +class IterToGlobalEnd[CC[_], C] extends ToGlobalEnd[({type X[A] = IterableOnceOps[A, CC, C]})#X, IterTypes[CC, C]] { + override def apply[I, O](in: IN[I])(f: Seq[I] => O): O = { + f(in.toSeq) + } }