Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[GLUTEN-8010][CORE] Don't generate native metrics if transformer don't generate relNode #8011

Merged
merged 8 commits into from
Nov 25, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,9 @@ abstract class FilterExecTransformerBase(val cond: Expression, val input: SparkP
case _ => false
}

override def metricsUpdater(): MetricsUpdater = if (getRemainingCondition == null) {
override def isLoop: Boolean = getRemainingCondition == null
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

s/isLoop/isNoop/


override def metricsUpdater(): MetricsUpdater = if (isLoop) {
MetricsUpdater.None
} else {
BackendsApiManager.getMetricsApiInstance.genFilterTransformerMetricsUpdater(metrics)
Expand Down Expand Up @@ -152,7 +154,7 @@ abstract class FilterExecTransformerBase(val cond: Expression, val input: SparkP

override protected def doTransform(context: SubstraitContext): TransformContext = {
val childCtx = child.asInstanceOf[TransformSupport].transform(context)
if (metricsUpdater == MetricsUpdater.None) {
if (isLoop) {
// The computing for this filter is not needed.
// Since some columns' nullability will be removed after this filter, we need to update the
// outputAttributes of child context.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,9 @@ case class ExpandExecTransformer(
AttributeSet.fromAttributeSets(projections.flatten.map(_.references))
}

override def metricsUpdater(): MetricsUpdater = if (projections == null || projections.isEmpty) {
override def isLoop: Boolean = projections == null || projections.isEmpty
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

s/isLoop/isNoop/


override def metricsUpdater(): MetricsUpdater = if (isLoop) {
MetricsUpdater.None
} else {
BackendsApiManager.getMetricsApiInstance.genExpandTransformerMetricsUpdater(metrics)
Expand Down Expand Up @@ -115,7 +117,7 @@ case class ExpandExecTransformer(

override protected def doTransform(context: SubstraitContext): TransformContext = {
val childCtx = child.asInstanceOf[TransformSupport].transform(context)
if (metricsUpdater == MetricsUpdater.None) {
if (isLoop) {
// The computing for this Expand is not needed.
return childCtx
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,9 @@ case class SortExecTransformer(
@transient override lazy val metrics =
BackendsApiManager.getMetricsApiInstance.genSortTransformerMetrics(sparkContext)

override def metricsUpdater(): MetricsUpdater = if (sortOrder == null || sortOrder.isEmpty) {
override def isLoop: Boolean = sortOrder == null || sortOrder.isEmpty
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

s/isLoop/isNoop/


override def metricsUpdater(): MetricsUpdater = if (isLoop) {
MetricsUpdater.None
} else {
BackendsApiManager.getMetricsApiInstance.genSortTransformerMetricsUpdater(metrics)
Expand Down Expand Up @@ -106,7 +108,7 @@ case class SortExecTransformer(

override protected def doTransform(context: SubstraitContext): TransformContext = {
val childCtx = child.asInstanceOf[TransformSupport].transform(context)
if (metricsUpdater == MetricsUpdater.None) {
if (isLoop) {
// The computing for this project is not needed.
return childCtx
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,9 @@ trait TransformSupport extends GlutenPlan {
Seq(plan.executeColumnar())
}
}

// When true, it will not generate relNode, nor will it generate native metrics.
def isLoop: Boolean = false
}

trait LeafTransformSupport extends TransformSupport with LeafExecNode {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ case class WindowExecTransformer(
@transient override lazy val metrics =
BackendsApiManager.getMetricsApiInstance.genWindowTransformerMetrics(sparkContext)

override def metricsUpdater(): MetricsUpdater = if (
windowExpression == null || windowExpression.isEmpty
) {
override def isLoop: Boolean = windowExpression == null || windowExpression.isEmpty
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

s/isLoop/isNoop/


override def metricsUpdater(): MetricsUpdater = if (isLoop) {
MetricsUpdater.None
} else {
BackendsApiManager.getMetricsApiInstance.genWindowTransformerMetricsUpdater(metrics)
Expand Down Expand Up @@ -182,7 +182,7 @@ case class WindowExecTransformer(

override protected def doTransform(context: SubstraitContext): TransformContext = {
val childCtx = child.asInstanceOf[TransformSupport].transform(context)
if (metricsUpdater == MetricsUpdater.None) {
if (isLoop) {
// The computing for this operator is not needed.
return childCtx
}
Expand Down
Loading