From 5a47548ddcd9a0041a8c49625f5be799ad3a6215 Mon Sep 17 00:00:00 2001 From: Yaliang Wang Date: Wed, 8 Aug 2018 23:43:32 -0700 Subject: [PATCH] Initial commit to support nested field pruning --- .../presto/hive/HiveClientConfig.java | 13 ++ .../presto/hive/HiveColumnHandle.java | 24 +++- .../facebook/presto/hive/HiveMetadata.java | 47 +++++-- .../presto/hive/HivePageSourceProvider.java | 1 + .../presto/hive/HiveSessionProperties.java | 11 ++ .../presto/hive/orc/OrcPageSourceFactory.java | 2 +- .../parquet/ParquetPageSourceFactory.java | 5 +- .../presto/hive/parquet/ParquetTypeUtils.java | 52 ++++++++ .../presto/server/ServerMainModule.java | 3 + .../sql/planner/LookupSymbolResolver.java | 10 +- .../presto/sql/planner/PlanFragmenter.java | 19 ++- .../presto/sql/planner/PlanOptimizers.java | 28 +++++ .../facebook/presto/sql/planner/Symbol.java | 118 ++++++++++++++++-- .../sql/planner/SymbolToInputRewriter.java | 8 ++ .../presto/sql/planner/SymbolsExtractor.java | 46 +++++++ .../iterative/rule/PickColumnLayouts.java | 105 ++++++++++++++++ .../iterative/rule/PickTableLayout.java | 2 + .../rule/ProjectOffPushDownFieldRule.java | 90 +++++++++++++ .../rule/PruneFilterColumnFields.java | 62 +++++++++ .../rule/PruneProjectColumnFields.java | 48 +++++++ .../rule/PruneTableScanColumnFields.java | 75 +++++++++++ .../RemoveRedundantIdentityProjections.java | 4 + .../sql/planner/iterative/rule/Util.java | 50 +++++++- .../presto/sql/planner/plan/Assignments.java | 28 +++++ .../sanity/ValidateDependenciesChecker.java | 19 ++- .../presto/sql/tree/SymbolReference.java | 20 ++- .../com/facebook/presto/spi/Constraint.java | 30 +++++ .../presto/spi/predicate/FieldSet.java | 45 +++++++ 28 files changed, 927 insertions(+), 38 deletions(-) create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PickColumnLayouts.java create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ProjectOffPushDownFieldRule.java create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneFilterColumnFields.java create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneProjectColumnFields.java create mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneTableScanColumnFields.java create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/predicate/FieldSet.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java index 0dc5c97d7447..6a6475a07e1f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java @@ -99,6 +99,7 @@ public class HiveClientConfig private boolean useParquetColumnNames; private boolean parquetOptimizedReaderEnabled; private boolean parquetPredicatePushdownEnabled; + private boolean parquetNestedFieldsProjectionPushdownEnabled; private boolean assumeCanonicalPartitionKeys; @@ -675,6 +676,18 @@ public HiveClientConfig setParquetPredicatePushdownEnabled(boolean parquetPredic return this; } + public boolean isParquetNestedFieldsProjectionPushdownEnabled() + { + return parquetNestedFieldsProjectionPushdownEnabled; + } + + @Config("hive.parquet-nested-fields-projection-pushdown.enabled") + public HiveClientConfig setParquetNestedFieldsProjectionPushdownEnabled(boolean parquetNestedFieldsProjectionPushdownEnabled) + { + this.parquetNestedFieldsProjectionPushdownEnabled = parquetNestedFieldsProjectionPushdownEnabled; + return this; + } + @Deprecated public boolean isParquetOptimizedReaderEnabled() { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveColumnHandle.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveColumnHandle.java index 0b153e7f7181..e9f3d0ac2896 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveColumnHandle.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveColumnHandle.java @@ -15,6 +15,7 @@ import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.predicate.FieldSet; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.TypeSignature; import com.fasterxml.jackson.annotation.JsonCreator; @@ -56,15 +57,22 @@ public enum ColumnType } private final String name; + private final Optional
fieldSet; private final HiveType hiveType; private final TypeSignature typeName; private final int hiveColumnIndex; private final ColumnType columnType; private final Optional comment; + public HiveColumnHandle(String name, HiveType hiveType, TypeSignature typeSignature, int hiveColumnIndex, ColumnType columnType, Optional comment) + { + this(name, Optional.empty(), hiveType, typeSignature, hiveColumnIndex, columnType, comment); + } + @JsonCreator public HiveColumnHandle( @JsonProperty("name") String name, + @JsonProperty("fieldSet") Optional
fieldSet, @JsonProperty("hiveType") HiveType hiveType, @JsonProperty("typeSignature") TypeSignature typeSignature, @JsonProperty("hiveColumnIndex") int hiveColumnIndex, @@ -72,6 +80,7 @@ public HiveColumnHandle( @JsonProperty("comment") Optional comment) { this.name = requireNonNull(name, "name is null"); + this.fieldSet = requireNonNull(fieldSet, "fieldSet is null"); checkArgument(hiveColumnIndex >= 0 || columnType == PARTITION_KEY || columnType == SYNTHESIZED, "hiveColumnIndex is negative"); this.hiveColumnIndex = hiveColumnIndex; this.hiveType = requireNonNull(hiveType, "hiveType is null"); @@ -86,6 +95,12 @@ public String getName() return name; } + @JsonProperty + public Optional
getFieldSet() + { + return fieldSet; + } + @JsonProperty public HiveType getHiveType() { @@ -134,7 +149,7 @@ public ColumnType getColumnType() @Override public int hashCode() { - return Objects.hash(name, hiveColumnIndex, hiveType, columnType, comment); + return Objects.hash(name, fieldSet, hiveColumnIndex, hiveType, columnType, comment); } @Override @@ -148,6 +163,7 @@ public boolean equals(Object obj) } HiveColumnHandle other = (HiveColumnHandle) obj; return Objects.equals(this.name, other.name) && + Objects.equals(this.fieldSet, other.fieldSet) && Objects.equals(this.hiveColumnIndex, other.hiveColumnIndex) && Objects.equals(this.hiveType, other.hiveType) && Objects.equals(this.columnType, other.columnType) && @@ -159,6 +175,7 @@ public String toString() { return toStringHelper(this) .add("name", name) + .add("fieldSet", fieldSet.orElse(null)) .add("hiveType", hiveType) .add("hiveColumnIndex", hiveColumnIndex) .add("columnType", columnType) @@ -202,4 +219,9 @@ public static boolean isBucketColumnHandle(HiveColumnHandle column) { return column.getHiveColumnIndex() == BUCKET_COLUMN_INDEX; } + + public static HiveColumnHandle withFieldSet(HiveColumnHandle column, Optional
fieldSet) + { + return new HiveColumnHandle(column.name, fieldSet, column.getHiveType(), column.getTypeSignature(), column.getHiveColumnIndex(), column.getColumnType(), column.getComment()); + } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java index 1eb3aaafcd33..2768a5f1bbfb 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java @@ -107,6 +107,7 @@ import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.SYNTHESIZED; import static com.facebook.presto.hive.HiveColumnHandle.PATH_COLUMN_NAME; import static com.facebook.presto.hive.HiveColumnHandle.updateRowIdHandle; +import static com.facebook.presto.hive.HiveColumnHandle.withFieldSet; import static com.facebook.presto.hive.HiveErrorCode.HIVE_COLUMN_ORDER_MISMATCH; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CONCURRENT_MODIFICATION_DETECTED; import static com.facebook.presto.hive.HiveErrorCode.HIVE_EXCEEDED_PARTITION_LIMIT; @@ -1302,22 +1303,48 @@ public boolean supportsMetadataDelete(ConnectorSession session, ConnectorTableHa public List getTableLayouts(ConnectorSession session, ConnectorTableHandle tableHandle, Constraint constraint, Optional> desiredColumns) { HiveTableHandle handle = (HiveTableHandle) tableHandle; + System.err.println("+++++++HiveMetadata::getTableLayouts+++++++"); + System.err.println(constraint.getFieldSets()); HivePartitionResult hivePartitionResult = partitionManager.getPartitions(metastore, tableHandle, constraint); + HiveTableLayoutHandle layoutHandle = new HiveTableLayoutHandle( + handle.getSchemaTableName(), + ImmutableList.copyOf(hivePartitionResult.getPartitionColumns()), + getPartitionsAsList(hivePartitionResult), + hivePartitionResult.getCompactEffectivePredicate(), + hivePartitionResult.getEnforcedConstraint(), + hivePartitionResult.getBucketHandle(), + hivePartitionResult.getBucketFilter()); + + if (constraint.getFieldSets().isPresent()) { + return ImmutableList.of(new ConnectorTableLayoutResult( + pruneColumnFields(layoutHandle, constraint), + constraint.getSummary())); + } + return ImmutableList.of(new ConnectorTableLayoutResult( - getTableLayout( - session, - new HiveTableLayoutHandle( - handle.getSchemaTableName(), - ImmutableList.copyOf(hivePartitionResult.getPartitionColumns()), - getPartitionsAsList(hivePartitionResult), - hivePartitionResult.getCompactEffectivePredicate(), - hivePartitionResult.getEnforcedConstraint(), - hivePartitionResult.getBucketHandle(), - hivePartitionResult.getBucketFilter())), + getTableLayout(session, layoutHandle), hivePartitionResult.getUnenforcedConstraint())); } + private ConnectorTableLayout pruneColumnFields(HiveTableLayoutHandle layoutHandle, Constraint constraint) + { + Optional> columns = constraint.getFieldSets() + .map(fieldsSets -> fieldsSets.stream() + .filter(entry -> !((HiveColumnHandle) entry.getKey()).getFieldSet().isPresent()) + .map(entry -> withFieldSet((HiveColumnHandle) entry.getKey(), Optional.of(entry.getValue()))) + .collect(toImmutableList())); + + return new ConnectorTableLayout( + layoutHandle, + columns, + TupleDomain.all(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + emptyList()); + } + @Override public ConnectorTableLayout getTableLayout(ConnectorSession session, ConnectorTableLayoutHandle layoutHandle) { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java index a8ea96480e6d..6d2bc4b37382 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java @@ -356,6 +356,7 @@ public static List toColumnHandles(List regular } return new HiveColumnHandle( columnHandle.getName(), + columnHandle.getFieldSet(), columnMapping.getCoercionFrom().get(), columnMapping.getCoercionFrom().get().getTypeSignature(), columnHandle.getHiveColumnIndex(), diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java index 9283da1fa619..88a4b2368e5e 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java @@ -49,6 +49,7 @@ public final class HiveSessionProperties private static final String RESPECT_TABLE_FORMAT = "respect_table_format"; private static final String PARQUET_PREDICATE_PUSHDOWN_ENABLED = "parquet_predicate_pushdown_enabled"; private static final String PARQUET_OPTIMIZED_READER_ENABLED = "parquet_optimized_reader_enabled"; + private static final String PARQUET_NESTED_FIELDS_PROJECTION_PUSHDOWN_READER_ENABLED = "parquet_nested_fields_projection_pushdown_enabled"; private static final String MAX_SPLIT_SIZE = "max_split_size"; private static final String MAX_INITIAL_SPLIT_SIZE = "max_initial_split_size"; public static final String RCFILE_OPTIMIZED_WRITER_ENABLED = "rcfile_optimized_writer_enabled"; @@ -153,6 +154,11 @@ public HiveSessionProperties(HiveClientConfig hiveClientConfig, OrcFileWriterCon "Experimental: Parquet: Enable predicate pushdown for Parquet", hiveClientConfig.isParquetPredicatePushdownEnabled(), false), + booleanSessionProperty( + PARQUET_NESTED_FIELDS_PROJECTION_PUSHDOWN_READER_ENABLED, + "Experimental: Parquet: Enable nested fields projection pushdown for Parquet", + hiveClientConfig.isParquetNestedFieldsProjectionPushdownEnabled(), + false), dataSizeSessionProperty( MAX_SPLIT_SIZE, "Max split size", @@ -299,6 +305,11 @@ public static boolean isParquetPredicatePushdownEnabled(ConnectorSession session return session.getProperty(PARQUET_PREDICATE_PUSHDOWN_ENABLED, Boolean.class); } + public static boolean isParquetNestedFieldsProjectionPushdownEnabled(ConnectorSession session) + { + return session.getProperty(PARQUET_NESTED_FIELDS_PROJECTION_PUSHDOWN_READER_ENABLED, Boolean.class); + } + public static DataSize getMaxSplitSize(ConnectorSession session) { return session.getProperty(MAX_SPLIT_SIZE, DataSize.class); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcPageSourceFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcPageSourceFactory.java index 4fe3fe0b43d3..33783f38d20c 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcPageSourceFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcPageSourceFactory.java @@ -261,7 +261,7 @@ private static List getPhysicalHiveColumnHandles(List fields = columns.stream() .filter(column -> column.getColumnType() == REGULAR) - .map(column -> getParquetType(column, fileSchema, useParquetColumnNames)) + .map(column -> getPrunedParquetType(column, fileSchema, useParquetColumnNames, isParquetNestedFieldsProjectionPushdownEnabled(session))) .filter(Objects::nonNull) .collect(toList()); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java index ad03201063ea..a348be84f89d 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java @@ -24,6 +24,8 @@ import com.facebook.presto.spi.type.TimestampType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.VarcharType; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import parquet.column.Encoding; import parquet.io.ColumnIO; import parquet.io.ColumnIOFactory; @@ -37,12 +39,15 @@ import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Optional.empty; import static parquet.schema.OriginalType.DECIMAL; import static parquet.schema.Type.Repetition.REPEATED; @@ -194,6 +199,53 @@ public static int getFieldIndex(MessageType fileSchema, String name) } } + public static parquet.schema.Type getPrunedParquetType(HiveColumnHandle column, MessageType messageType, boolean useParquetColumnNames, boolean pruneNestedFields) + { + parquet.schema.Type originalType = getParquetType(column, messageType, useParquetColumnNames); + if (pruneNestedFields && column.getFieldSet().isPresent()) { + return pruneParquetType(originalType, column.getFieldSet().get().getFields()); + } + + return originalType; + } + + private static parquet.schema.Type pruneParquetType(parquet.schema.Type type, Set requiredFields) + { + if (requiredFields.isEmpty()) { + return type; + } + + if (type.isPrimitive()) { + return type; + } + + Map> fields = groupFields(requiredFields); + + List newFields = fields.entrySet().stream() + .map(entry -> pruneParquetType(type.asGroupType().getType(entry.getKey()), entry.getValue())) + .collect(toImmutableList()); + + return type.asGroupType().withNewFields(newFields); + } + + private static Map> groupFields(Set requiredFields) + { + Map> fields = new HashMap<>(); + for (String field : requiredFields) { + String[] path = field.split("\\.", 2); + String fieldName = path[0]; + Set nestedField = path.length == 1 ? ImmutableSet.of() : ImmutableSet.of(path[1]); + if (fields.containsKey(fieldName)) { + fields.get(fieldName).addAll(nestedField); + } + else { + fields.put(fieldName, new HashSet<>(nestedField)); + } + } + + return ImmutableMap.copyOf(fields); + } + public static parquet.schema.Type getParquetType(HiveColumnHandle column, MessageType messageType, boolean useParquetColumnNames) { if (useParquetColumnNames) { diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java index 6e2dcce9cad6..b60c1bee03eb 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java @@ -140,6 +140,7 @@ import com.facebook.presto.sql.planner.LocalExecutionPlanner; import com.facebook.presto.sql.planner.NodePartitioningManager; import com.facebook.presto.sql.planner.PlanOptimizers; +import com.facebook.presto.sql.planner.Symbol; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.FunctionCall; import com.facebook.presto.transaction.ForTransactionManager; @@ -434,6 +435,8 @@ protected void setup(Binder binder) jsonBinder(binder).addSerializerBinding(Expression.class).to(ExpressionSerializer.class); jsonBinder(binder).addDeserializerBinding(Expression.class).to(ExpressionDeserializer.class); jsonBinder(binder).addDeserializerBinding(FunctionCall.class).to(FunctionCallDeserializer.class); + jsonBinder(binder).addKeySerializerBinding(Symbol.class).to(Symbol.SymbolKeySerializer.class); + jsonBinder(binder).addKeyDeserializerBinding(Symbol.class).to(Symbol.SymbolKeyDeserializer.class); // query monitor configBinder(binder).bindConfig(QueryMonitorConfig.class); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LookupSymbolResolver.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LookupSymbolResolver.java index dcd0ef41cd0c..d0b6cdbc82d6 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LookupSymbolResolver.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LookupSymbolResolver.java @@ -20,11 +20,13 @@ import java.util.Map; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static java.util.Objects.requireNonNull; public class LookupSymbolResolver implements SymbolResolver { + private final Map assignmentSymbolLookup; private final Map assignments; private final Map bindings; @@ -33,6 +35,8 @@ public LookupSymbolResolver(Map assignments, Map symbol)); this.assignments = ImmutableMap.copyOf(assignments); this.bindings = ImmutableMap.copyOf(bindings); } @@ -40,11 +44,13 @@ public LookupSymbolResolver(Map assignments, Map typeMapper(Set dependencies) + { + ImmutableMap.Builder builder = ImmutableMap.builder(); + for (Symbol symbol : dependencies) { + if (types.containsKey(symbol)) { + builder.put(symbol, types.get(symbol)); + continue; + } + if (types.containsKey(new Symbol(symbol.getName()))) { + builder.put(symbol, types.get(new Symbol(symbol.getName()))); + } + } + return builder.build(); + } + @Override public PlanNode visitOutput(OutputNode node, RewriteContext context) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java index e262c48c27fe..70a982ce7903 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java @@ -43,11 +43,13 @@ import com.facebook.presto.sql.planner.iterative.rule.MergeLimitWithTopN; import com.facebook.presto.sql.planner.iterative.rule.MergeLimits; import com.facebook.presto.sql.planner.iterative.rule.MultipleDistinctAggregationToMarkDistinct; +import com.facebook.presto.sql.planner.iterative.rule.PickColumnLayouts; import com.facebook.presto.sql.planner.iterative.rule.PickTableLayout; import com.facebook.presto.sql.planner.iterative.rule.PruneAggregationColumns; import com.facebook.presto.sql.planner.iterative.rule.PruneAggregationSourceColumns; import com.facebook.presto.sql.planner.iterative.rule.PruneCountAggregationOverScalar; import com.facebook.presto.sql.planner.iterative.rule.PruneCrossJoinColumns; +import com.facebook.presto.sql.planner.iterative.rule.PruneFilterColumnFields; import com.facebook.presto.sql.planner.iterative.rule.PruneFilterColumns; import com.facebook.presto.sql.planner.iterative.rule.PruneIndexSourceColumns; import com.facebook.presto.sql.planner.iterative.rule.PruneJoinChildrenColumns; @@ -56,9 +58,11 @@ import com.facebook.presto.sql.planner.iterative.rule.PruneMarkDistinctColumns; import com.facebook.presto.sql.planner.iterative.rule.PruneOrderByInAggregation; import com.facebook.presto.sql.planner.iterative.rule.PruneOutputColumns; +import com.facebook.presto.sql.planner.iterative.rule.PruneProjectColumnFields; import com.facebook.presto.sql.planner.iterative.rule.PruneProjectColumns; import com.facebook.presto.sql.planner.iterative.rule.PruneSemiJoinColumns; import com.facebook.presto.sql.planner.iterative.rule.PruneSemiJoinFilteringSourceColumns; +import com.facebook.presto.sql.planner.iterative.rule.PruneTableScanColumnFields; import com.facebook.presto.sql.planner.iterative.rule.PruneTableScanColumns; import com.facebook.presto.sql.planner.iterative.rule.PruneTopNColumns; import com.facebook.presto.sql.planner.iterative.rule.PruneValuesColumns; @@ -197,6 +201,11 @@ public PlanOptimizers( new PruneLimitColumns(), new PruneTableScanColumns()); + Set> fieldPruningRules = ImmutableSet.of( + new PruneProjectColumnFields(), + new PruneFilterColumnFields(), + new PruneTableScanColumnFields()); + IterativeOptimizer inlineProjections = new IterativeOptimizer( stats, statsCalculator, @@ -415,6 +424,25 @@ public PlanOptimizers( .add(new InlineProjections()) .build())); + // Optimize the nested fields. + builder.add(new IterativeOptimizer( + stats, + statsCalculator, + costCalculator, + ImmutableSet.>builder() + .addAll(fieldPruningRules) + .add(new RemoveRedundantIdentityProjections()) + .build())); + + // Update the table layouts after pruning. + builder.add(new IterativeOptimizer( + stats, + statsCalculator, + costCalculator, + ImmutableSet.>builder() + .add(new PickColumnLayouts(metadata)) + .build())); + // Optimizers above this don't understand local exchanges, so be careful moving this. builder.add(new AddLocalExchanges(metadata, sqlParser)); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/Symbol.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/Symbol.java index 8168d4b4ef6f..ae0dd4f3a5ee 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/Symbol.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/Symbol.java @@ -13,11 +13,25 @@ */ package com.facebook.presto.sql.planner; +import com.facebook.presto.sql.tree.DereferenceExpression; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.SymbolReference; import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonValue; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.KeyDeserializer; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.google.common.collect.ImmutableSet; +import io.airlift.json.ObjectMapperProvider; +import java.io.IOException; +import java.util.Objects; +import java.util.Set; + +import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; @@ -25,35 +39,84 @@ public class Symbol implements Comparable { private final String name; + private final Set fields; public static Symbol from(Expression expression) { checkArgument(expression instanceof SymbolReference, "Unexpected expression: %s", expression); - return new Symbol(((SymbolReference) expression).getName()); + return new Symbol(((SymbolReference) expression).getName(), ((SymbolReference) expression).getFields()); + } + + public static Symbol withField(DereferenceExpression expression) + { + checkArgument(expression.getBase() instanceof SymbolReference, "Unexpected base expression: %s", expression.getBase()); + return new Symbol(((SymbolReference) expression.getBase()).getName(), ImmutableSet.of(expression.getField().getValue())); + } + + public static Symbol withAllFields(SymbolReference expression) + { + return new Symbol(expression.getName(), expression.getFields()); + } + + public static Symbol merge(Symbol a, Symbol b) + { + checkArgument(a.getName().equalsIgnoreCase(b.getName()), "Symbols with different names cannot be merged: %s vs %s", a.getName(), b.getName()); + return new Symbol(a.getName(), mergeAndPruneFields(a.fields, b.fields)); + } + + public static boolean contains(Symbol a, Symbol b) + { + if (!a.getName().equalsIgnoreCase(b.getName())) { + return false; + } + + if (!a.getFields().isEmpty() && !a.getFields().containsAll(b.getFields())) { + return false; + } + + return true; } - @JsonCreator public Symbol(String name) + { + this(name, ImmutableSet.of()); + } + + @JsonCreator + public Symbol( + @JsonProperty("name") String name, + @JsonProperty("fields") Set fields) { requireNonNull(name, "name is null"); + requireNonNull(fields, "fields is null"); this.name = name; + this.fields = fields; } - @JsonValue + @JsonProperty public String getName() { return name; } + @JsonProperty + public Set getFields() + { + return fields; + } + public SymbolReference toSymbolReference() { - return new SymbolReference(name); + return new SymbolReference(name, fields); } @Override public String toString() { - return name; + return toStringHelper(this) + .add("name", name) + .add("fields", fields) + .toString(); } @Override @@ -72,13 +135,17 @@ public boolean equals(Object o) return false; } + if (!fields.equals(symbol.fields)) { + return false; + } + return true; } @Override public int hashCode() { - return name.hashCode(); + return Objects.hash(name, fields); } @Override @@ -86,4 +153,41 @@ public int compareTo(Symbol o) { return name.compareTo(o.name); } + + private static Set mergeAndPruneFields(Set a, Set b) + { + if (a.size() == 0 || b.size() == 0) { + return ImmutableSet.of(); + } + return ImmutableSet.builder() + .addAll(a) + .addAll(b) + .build(); + } + + public static class SymbolKeySerializer + extends JsonSerializer + { + private static final ObjectMapper MAPPER = new ObjectMapperProvider().get(); + + @Override + public void serialize(Symbol symbol, JsonGenerator jsonGenerator, SerializerProvider serializers) + throws IOException + { + jsonGenerator.writeFieldName(MAPPER.writeValueAsString(symbol)); + } + } + + public static class SymbolKeyDeserializer + extends KeyDeserializer + { + private static final ObjectMapper MAPPER = new ObjectMapperProvider().get(); + + @Override + public Object deserializeKey(String key, DeserializationContext ctxt) + throws IOException + { + return MAPPER.readValue(key, Symbol.class); + } + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolToInputRewriter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolToInputRewriter.java index e965af2a8816..bbe679bba1fc 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolToInputRewriter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolToInputRewriter.java @@ -24,16 +24,20 @@ import java.util.Map; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static java.util.Objects.requireNonNull; public class SymbolToInputRewriter { private final Map symbolToChannelMapping; + private final Map nameToSymbolMapping; public SymbolToInputRewriter(Map symbolToChannelMapping) { requireNonNull(symbolToChannelMapping, "symbolToChannelMapping is null"); this.symbolToChannelMapping = ImmutableMap.copyOf(symbolToChannelMapping); + this.nameToSymbolMapping = symbolToChannelMapping.keySet().stream() + .collect(toImmutableMap(Symbol::getName, symbol -> symbol)); } public Expression rewrite(Expression expression) @@ -44,6 +48,10 @@ public Expression rewrite(Expression expression) public Expression rewriteSymbolReference(SymbolReference node, Context context, ExpressionTreeRewriter treeRewriter) { Integer channel = symbolToChannelMapping.get(Symbol.from(node)); + if (channel == null) { + // todo: may need to also check fields + channel = symbolToChannelMapping.get(nameToSymbolMapping.get(node.getName())); + } if (channel == null) { Preconditions.checkArgument(context.isInLambda(), "Cannot resolve symbol %s", node.getName()); return node; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolsExtractor.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolsExtractor.java index c04347ee11b8..579cc7d470ca 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolsExtractor.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolsExtractor.java @@ -24,9 +24,12 @@ import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.SymbolReference; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import static com.facebook.presto.sql.planner.ExpressionExtractor.extractExpressions; @@ -107,6 +110,31 @@ public static Set extractOutputSymbols(PlanNode planNode, Lookup lookup) .collect(toImmutableSet()); } + public static Map extractUniqueWithFields(Iterable expressions) + { + Map symbolMap = new HashMap<>(); + for (Expression expression : expressions) { + extractAllWithFields(expression).forEach(symbol -> { + String name = symbol.getName(); + if (symbolMap.containsKey(name)) { + symbolMap.put(name, Symbol.merge(symbolMap.get(name), symbol)); + } + else { + symbolMap.put(name, symbol); + } + }); + } + + return ImmutableMap.copyOf(symbolMap); + } + + public static List extractAllWithFields(Expression expression) + { + ImmutableList.Builder builder = ImmutableList.builder(); + new SymbolWithFieldsBuilderVisitor().process(expression, builder); + return builder.build(); + } + private static class SymbolBuilderVisitor extends DefaultExpressionTraversalVisitor> { @@ -147,4 +175,22 @@ protected Void visitIdentifier(Identifier node, ImmutableSet.Builder> + { + @Override + protected Void visitDereferenceExpression(DereferenceExpression expression, ImmutableList.Builder builder) + { + builder.add(Symbol.withField(expression)); + return null; + } + + @Override + protected Void visitSymbolReference(SymbolReference node, ImmutableList.Builder builder) + { + builder.add(Symbol.withAllFields(node)); + return null; + } + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PickColumnLayouts.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PickColumnLayouts.java new file mode 100644 index 000000000000..daf2bb15c593 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PickColumnLayouts.java @@ -0,0 +1,105 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.metadata.TableLayoutResult; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.Constraint; +import com.facebook.presto.spi.predicate.FieldSet; +import com.facebook.presto.sql.planner.Symbol; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.facebook.presto.sql.planner.plan.TableScanNode; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.sql.planner.plan.Patterns.tableScan; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static java.util.Objects.requireNonNull; + +public class PickColumnLayouts + implements Rule +{ + private final Metadata metadata; + private static final Pattern PATTERN = tableScan(); + + public PickColumnLayouts(Metadata metadata) + { + this.metadata = requireNonNull(metadata, "metadata is null"); + } + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public Result apply(TableScanNode node, Captures captures, Context context) + { + System.err.println("##############PickColumnLayouts:apply#############"); + System.err.println(node.getAssignments()); + + List> fieldSets = node.getAssignments().keySet().stream() + .filter(symbol -> !symbol.getFields().isEmpty()) + .map(symbol -> Maps.immutableEntry(node.getAssignments().get(symbol), new FieldSet(symbol.getFields()))) + .collect(toImmutableList()); + + ImmutableMap.Builder builder = ImmutableMap.builder(); + for (int i = 0; i < fieldSets.size(); i++) { + builder.put(fieldSets.get(i).getKey(), i); + } + Map columnIndex = builder.build(); + + if (fieldSets.isEmpty()) { + return Result.empty(); + } + + List layouts = metadata.getLayouts( + context.getSession(), + node.getTable(), + new Constraint<>(Optional.of(fieldSets)), + Optional.of(ImmutableSet.copyOf(node.getAssignments().values()))); + + TableLayoutResult layout = layouts.get(0); + + if (!layout.getLayout().getColumns().isPresent() || layout.getLayout().getColumns().get().isEmpty()) { + return Result.empty(); + } + + List columns = layout.getLayout().getColumns().get(); + + Map assignments = node.getAssignments().keySet().stream() + .filter(symbol -> !symbol.getFields().isEmpty()) + .map(symbol -> Maps.immutableEntry(symbol, columns.get(columnIndex.get(node.getAssignments().get(symbol))))) + .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); + + return Result.ofPlanNode(new TableScanNode( + node.getId(), + node.getTable(), + node.getOutputSymbols(), + node.getAssignments().keySet().stream().map(symbol -> Maps.immutableEntry(symbol, assignments.getOrDefault(symbol, node.getAssignments().get(symbol)))).collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)), + node.getLayout(), + node.getCurrentConstraint(), + node.getOriginalConstraint())); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PickTableLayout.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PickTableLayout.java index 17d73405857e..f814d5a618e4 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PickTableLayout.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PickTableLayout.java @@ -189,6 +189,8 @@ public Result apply(TableScanNode tableScanNode, Captures captures, Context cont private static PlanNode planTableScan(TableScanNode node, Expression predicate, Rule.Context context, Metadata metadata, DomainTranslator domainTranslator) { + System.err.println("##############planTableScan#############"); + System.err.println(node.getAssignments()); Expression deterministicPredicate = filterDeterministicConjuncts(predicate); DomainTranslator.ExtractionResult decomposedPredicate = DomainTranslator.fromPredicate( metadata, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ProjectOffPushDownFieldRule.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ProjectOffPushDownFieldRule.java new file mode 100644 index 000000000000..2dcd115c29f4 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ProjectOffPushDownFieldRule.java @@ -0,0 +1,90 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.matching.Capture; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.sql.planner.PlanNodeIdAllocator; +import com.facebook.presto.sql.planner.Symbol; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.facebook.presto.sql.planner.plan.PlanNode; +import com.facebook.presto.sql.planner.plan.ProjectNode; +import com.google.common.collect.ImmutableList; + +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.matching.Capture.newCapture; +import static com.facebook.presto.sql.planner.iterative.rule.Util.pruneNestedFields; +import static com.facebook.presto.sql.planner.plan.Patterns.project; +import static com.facebook.presto.sql.planner.plan.Patterns.source; + +public abstract class ProjectOffPushDownFieldRule + implements Rule +{ + private final Capture targetCapture = newCapture(); + + private final Pattern targetPattern; + + protected ProjectOffPushDownFieldRule(Pattern targetPattern) + { + this.targetPattern = targetPattern; + } + + @Override + public Pattern getPattern() + { + return project() + .with(source().matching(targetPattern.capturedAs(targetCapture))); + } + + @Override + public Result apply(ProjectNode parent, Captures captures, Context context) + { + N targetNode = captures.get(targetCapture); + + System.err.println("----ProjectOffPushDownFieldRule::apply----"); + System.err.println(parent.getAssignments().getExpressions()); + + return pruneNestedFields(targetNode.getOutputSymbols(), parent.getAssignments().getExpressions()) + .map(prunedOutputs -> { + System.err.println("----process prunedOutputs----"); + System.err.println(targetNode.getOutputSymbols()); + System.err.println(prunedOutputs); + return prunedOutputs; + }) + .flatMap(prunedOutputs -> this.pushDownProjectOff(context.getIdAllocator(), targetNode, prunedOutputs)) + .map(newChild -> { + System.err.println("----process newChild----"); + System.err.println(newChild.getClass().getName()); + System.err.println(newChild.getId()); + System.err.println(newChild.getOutputSymbols()); + return newChild; + }) + .map(newChild -> parent.replaceChildren(ImmutableList.of(newChild))) + .map(newParent -> { + System.err.println("----process newParent----"); + System.err.println(newParent.getClass().getName()); + System.err.println(newParent.getId()); + System.err.println(newParent.getOutputSymbols()); + System.err.println(((ProjectNode) newParent).getAssignments()); + return newParent; + }) + .map(Result::ofPlanNode) + .orElse(Result.empty()); + } + + protected abstract Optional pushDownProjectOff(PlanNodeIdAllocator idAllocator, N targetNode, Set referencedOutputs); +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneFilterColumnFields.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneFilterColumnFields.java new file mode 100644 index 000000000000..680b1f23199e --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneFilterColumnFields.java @@ -0,0 +1,62 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.sql.planner.PlanNodeIdAllocator; +import com.facebook.presto.sql.planner.Symbol; +import com.facebook.presto.sql.planner.plan.FilterNode; +import com.facebook.presto.sql.planner.plan.PlanNode; +import com.google.common.collect.ImmutableSet; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.sql.planner.SymbolsExtractor.extractUniqueWithFields; +import static com.facebook.presto.sql.planner.iterative.rule.Util.restrictChildOutputs; +import static com.facebook.presto.sql.planner.plan.Patterns.filter; + +public class PruneFilterColumnFields + extends ProjectOffPushDownFieldRule +{ + public PruneFilterColumnFields() + { + super(filter()); + } + + @Override + protected Optional pushDownProjectOff(PlanNodeIdAllocator idAllocator, FilterNode filterNode, Set referencedOutputs) + { + System.err.println("----PruneFilterColumns::referencedOutputs----"); + System.err.println(referencedOutputs); + Map filterInputsMap = new HashMap<>(extractUniqueWithFields(ImmutableSet.of(filterNode.getPredicate()))); + + referencedOutputs.forEach(symbol -> { + String name = symbol.getName(); + if (filterInputsMap.containsKey(name)) { + filterInputsMap.put(name, Symbol.merge(filterInputsMap.get(name), symbol)); + } + else { + filterInputsMap.put(name, symbol); + } + }); + Set prunedFilterInputs = ImmutableSet.copyOf(filterInputsMap.values()); + + System.err.println("----prunedFilterInputs----"); + System.err.println(prunedFilterInputs); + + return restrictChildOutputs(idAllocator, filterNode, prunedFilterInputs); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneProjectColumnFields.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneProjectColumnFields.java new file mode 100644 index 000000000000..2c88de0a2c88 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneProjectColumnFields.java @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.sql.planner.PlanNodeIdAllocator; +import com.facebook.presto.sql.planner.Symbol; +import com.facebook.presto.sql.planner.plan.PlanNode; +import com.facebook.presto.sql.planner.plan.ProjectNode; + +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.sql.planner.plan.Patterns.project; + +public class PruneProjectColumnFields + extends ProjectOffPushDownFieldRule +{ + public PruneProjectColumnFields() + { + super(project()); + } + + @Override + protected Optional pushDownProjectOff( + PlanNodeIdAllocator idAllocator, + ProjectNode childProjectNode, + Set referencedOutputs) + { + System.err.println("----PruneProjectColumns::referencedOutputs----"); + System.err.println(referencedOutputs); + return Optional.of( + new ProjectNode( + childProjectNode.getId(), + childProjectNode.getSource(), + childProjectNode.getAssignments().refineIdentities(referencedOutputs))); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneTableScanColumnFields.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneTableScanColumnFields.java new file mode 100644 index 000000000000..a7855f103dcc --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneTableScanColumnFields.java @@ -0,0 +1,75 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.sql.planner.PlanNodeIdAllocator; +import com.facebook.presto.sql.planner.Symbol; +import com.facebook.presto.sql.planner.plan.PlanNode; +import com.facebook.presto.sql.planner.plan.TableScanNode; +import com.google.common.collect.Maps; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.sql.planner.plan.Patterns.tableScan; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; + +public class PruneTableScanColumnFields + extends ProjectOffPushDownFieldRule +{ + public PruneTableScanColumnFields() + { + super(tableScan()); + } + + @Override + protected Optional pushDownProjectOff(PlanNodeIdAllocator idAllocator, TableScanNode tableScanNode, Set referencedOutputs) + { + System.err.println("----PruneTableScanColumns::referencedOutputs----"); + System.err.println(referencedOutputs); + return Optional.of( + new TableScanNode( + tableScanNode.getId(), + tableScanNode.getTable(), + filteredCopy(tableScanNode.getOutputSymbols(), referencedOutputs), + filterKeys(tableScanNode.getAssignments(), referencedOutputs), + tableScanNode.getLayout(), + tableScanNode.getCurrentConstraint(), + tableScanNode.getOriginalConstraint())); + } + + private List filteredCopy(List outputSymbols, Set referencedOutputs) + { + Map referencedOutputsMap = referencedOutputs.stream() + .collect(toImmutableMap(Symbol::getName, symbol -> symbol)); + return outputSymbols.stream() + .filter(symbol -> referencedOutputsMap.containsKey(symbol.getName())) + .map(symbol -> referencedOutputsMap.get(symbol.getName())) + .collect(toImmutableList()); + } + + private Map filterKeys(Map assignements, Set referencedOutputs) + { + Map referencedOutputsMap = referencedOutputs.stream() + .collect(toImmutableMap(Symbol::getName, symbol -> symbol)); + return assignements.keySet().stream() + .filter(symbol -> referencedOutputsMap.containsKey(symbol.getName())) + .map(symbol -> Maps.immutableEntry(referencedOutputsMap.get(symbol.getName()), assignements.get(symbol))) + .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantIdentityProjections.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantIdentityProjections.java index 4f9fc038b01c..63e352f89a52 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantIdentityProjections.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantIdentityProjections.java @@ -47,6 +47,10 @@ public Pattern getPattern() @Override public Result apply(ProjectNode project, Captures captures, Context context) { + System.err.println("***********RemoveRedundantIdentityProjections:apply***********"); + System.err.println(project.getAssignments()); + System.err.println(project.getOutputSymbols()); + System.err.println(project.getId()); return Result.ofPlanNode(project.getSource()); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/Util.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/Util.java index 240d81dd73ca..f552ef6a1085 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/Util.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/Util.java @@ -25,12 +25,15 @@ import com.google.common.collect.Sets; import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableSet.toImmutableSet; class Util { @@ -55,6 +58,29 @@ public static Optional> pruneInputs(Collection availableInpu return Optional.of(prunedInputs); } + public static Optional> pruneNestedFields(Collection availableInputs, Collection expressions) + { + Set availableInputsSet = ImmutableSet.copyOf(availableInputs); + Map expressionInputsMap = SymbolsExtractor.extractUniqueWithFields(expressions); + + Set prunedInputs = availableInputsSet.stream() + .filter(symbol -> expressionInputsMap.containsKey(symbol.getName())) + .collect(toImmutableSet()); + + if (prunedInputs.stream().allMatch(symbol -> expressionInputsMap.get(symbol.getName()).getFields().equals(symbol.getFields()))) { + return Optional.empty(); + } + + Set fieldPrunedInputs = prunedInputs.stream() + .map(symbol -> expressionInputsMap.getOrDefault(symbol.getName(), symbol)) + .collect(toImmutableSet()); + + System.err.println("before nested field prune: " + prunedInputs.toString()); + System.err.println("after nested field prune: " + fieldPrunedInputs.toString()); + + return Optional.of(fieldPrunedInputs); + } + /** * Transforms a plan like P->C->X to C->P->X */ @@ -70,13 +96,25 @@ public static PlanNode transpose(PlanNode parent, PlanNode child) */ public static Optional restrictOutputs(PlanNodeIdAllocator idAllocator, PlanNode node, Set permittedOutputs) { + System.err.println("----restrictOutputs-----"); + System.err.println("----restrictOutputs:original outputs-----"); + System.err.println(node.getOutputSymbols()); + Map permittedOutputsMap = new HashMap<>(); + permittedOutputs.forEach(symbol -> permittedOutputsMap.put(symbol.getName(), symbol)); + + if (node.getOutputSymbols().stream().allMatch(symbol -> + permittedOutputsMap.containsKey(symbol.getName()) + && permittedOutputsMap.get(symbol.getName()).getFields().equals(symbol.getFields()))) { + return Optional.empty(); + } + List restrictedOutputs = node.getOutputSymbols().stream() - .filter(permittedOutputs::contains) + .filter(symbol -> permittedOutputsMap.containsKey(symbol.getName())) + .map(symbol -> permittedOutputsMap.get(symbol.getName())) .collect(toImmutableList()); - if (restrictedOutputs.size() == node.getOutputSymbols().size()) { - return Optional.empty(); - } + System.err.println("----restrictOutputs::restrictedOutputs"); + System.err.println(restrictedOutputs); return Optional.of( new ProjectNode( @@ -103,16 +141,20 @@ public static Optional restrictChildOutputs(PlanNodeIdAllocator idAllo ImmutableList.Builder newChildrenBuilder = ImmutableList.builder(); boolean rewroteChildren = false; + System.err.println("----restrictChildOutputs::trying to restrict children---"); for (int i = 0; i < node.getSources().size(); ++i) { PlanNode oldChild = node.getSources().get(i); Optional newChild = restrictOutputs(idAllocator, oldChild, permittedChildOutputs.get(i)); rewroteChildren |= newChild.isPresent(); + newChild.ifPresent(child -> System.err.println(child.getId())); newChildrenBuilder.add(newChild.orElse(oldChild)); } if (!rewroteChildren) { return Optional.empty(); } + + System.err.println("----restrictChildOutputs::restrict children changed---"); return Optional.of(node.replaceChildren(newChildrenBuilder.build())); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Assignments.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Assignments.java index 8b6eeafa1812..c6e2c6e08d37 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Assignments.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Assignments.java @@ -34,7 +34,9 @@ import java.util.function.Function; import java.util.stream.Collector; +import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static java.util.Arrays.asList; import static java.util.Objects.requireNonNull; @@ -91,6 +93,7 @@ public List getOutputs() } @JsonProperty("assignments") + public Map getMap() { return assignments; @@ -108,6 +111,23 @@ public Assignments rewrite(Function rewrite) .collect(toAssignments()); } + public Assignments refineIdentities(Collection symbols) + { + Map symbolMap = assignments.keySet().stream() + .map(symbol -> Maps.immutableEntry(symbol.getName(), symbol)) + .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); + + return symbols.stream() + .filter(symbol -> symbolMap.containsKey(symbol.getName())) + .map(symbol -> { + if (isIdentity(symbolMap.get(symbol.getName()))) { + return Maps.immutableEntry(symbol, (Expression) symbol.toSymbolReference()); + } + return Maps.immutableEntry(symbol, assignments.get(symbolMap.get(symbol.getName()))); + }) + .collect(toAssignments()); + } + public Assignments filter(Collection symbols) { return filter(symbols::contains); @@ -190,6 +210,14 @@ public int hashCode() return assignments.hashCode(); } + @Override + public String toString() + { + return toStringHelper(this) + .add("assignments", assignments) + .toString(); + } + public static class Builder { private final Map assignments = new LinkedHashMap<>(); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java index 64a641a560a9..ad271b6ff741 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java @@ -71,6 +71,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; /** @@ -223,8 +224,8 @@ public Void visitFilter(FilterNode node, Set boundSymbols) Set inputs = createInputs(source, boundSymbols); checkDependencies(inputs, node.getOutputSymbols(), "Invalid node. Output symbols (%s) not in source plan output (%s)", node.getOutputSymbols(), node.getSource().getOutputSymbols()); - Set dependencies = SymbolsExtractor.extractUnique(node.getPredicate()); - checkDependencies(inputs, dependencies, "Invalid node. Predicate dependencies (%s) not in source plan output (%s)", dependencies, node.getSource().getOutputSymbols()); + Collection dependencies = SymbolsExtractor.extractUniqueWithFields(ImmutableSet.of(node.getPredicate())).values(); + checkDependenciesWithFields(inputs, dependencies, "Invalid node. Predicate dependencies (%s) not in source plan output (%s)", dependencies, node.getSource().getOutputSymbols()); return null; } @@ -245,10 +246,8 @@ public Void visitProject(ProjectNode node, Set boundSymbols) source.accept(this, boundSymbols); // visit child Set inputs = createInputs(source, boundSymbols); - for (Expression expression : node.getAssignments().getExpressions()) { - Set dependencies = SymbolsExtractor.extractUnique(expression); - checkDependencies(inputs, dependencies, "Invalid node. Expression dependencies (%s) not in source plan output (%s)", dependencies, inputs); - } + Collection dependencies = SymbolsExtractor.extractUniqueWithFields(node.getAssignments().getExpressions()).values(); + checkDependenciesWithFields(inputs, dependencies, "Invalid node. Expression dependencies (%s) not in source plan output (%s)", dependencies, inputs); return null; } @@ -615,4 +614,12 @@ private static void checkDependencies(Collection inputs, Collection inputs, Collection required, String message, Object... parameters) + { + Map inputsMap = inputs.stream() + .collect(toImmutableMap(Symbol::getName, symbol -> symbol)); + checkArgument(required.stream() + .allMatch(symbol -> inputsMap.containsKey(symbol.getName()) && Symbol.contains(inputsMap.get(symbol.getName()), symbol)), message, parameters); + } } diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/SymbolReference.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/SymbolReference.java index 51b88d4dcf3a..69e6c1973a49 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/tree/SymbolReference.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/SymbolReference.java @@ -14,20 +14,31 @@ package com.facebook.presto.sql.tree; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; public class SymbolReference extends Expression { private final String name; + private final Set fields; public SymbolReference(String name) { super(Optional.empty()); this.name = name; + this.fields = ImmutableSet.of(); + } + + public SymbolReference(String name, Set fields) + { + super(Optional.empty()); + this.name = name; + this.fields = ImmutableSet.copyOf(fields); } public String getName() @@ -35,6 +46,11 @@ public String getName() return name; } + public Set getFields() + { + return fields; + } + @Override public R accept(AstVisitor visitor, C context) { @@ -57,12 +73,12 @@ public boolean equals(Object o) return false; } SymbolReference that = (SymbolReference) o; - return Objects.equals(name, that.name); + return Objects.equals(name, that.name) && Objects.equals(fields, that.fields); } @Override public int hashCode() { - return Objects.hash(name); + return Objects.hash(name, fields); } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/Constraint.java b/presto-spi/src/main/java/com/facebook/presto/spi/Constraint.java index b993eec7039d..36ad47b083de 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/Constraint.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/Constraint.java @@ -13,10 +13,13 @@ */ package com.facebook.presto.spi; +import com.facebook.presto.spi.predicate.FieldSet; import com.facebook.presto.spi.predicate.NullableValue; import com.facebook.presto.spi.predicate.TupleDomain; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.function.Predicate; import static java.util.Objects.requireNonNull; @@ -25,6 +28,7 @@ public class Constraint { private final TupleDomain summary; private final Predicate> predicate; + private final Optional>> fieldSets; public static Constraint alwaysTrue() { @@ -36,13 +40,25 @@ public static Constraint alwaysFalse() return new Constraint<>(TupleDomain.none(), bindings -> false); } + public Constraint(Optional>> fieldSets) + { + this(TupleDomain.all(), bindings -> true, fieldSets); + } + public Constraint(TupleDomain summary, Predicate> predicate) + { + this(summary, predicate, Optional.empty()); + } + + public Constraint(TupleDomain summary, Predicate> predicate, Optional>> fieldSets) { requireNonNull(summary, "summary is null"); requireNonNull(predicate, "predicate is null"); + requireNonNull(fieldSets, "fieldSets is null"); this.summary = summary; this.predicate = predicate; + this.fieldSets = fieldSets; } public TupleDomain getSummary() @@ -50,6 +66,20 @@ public TupleDomain getSummary() return summary; } + public Optional
getFieldSet(T key) + { + return fieldSets + .flatMap(fieldSets -> fieldSets.stream() + .filter(entry -> entry.getKey().equals(key)) + .findFirst()) + .map(Map.Entry::getValue); + } + + public Optional>> getFieldSets() + { + return fieldSets; + } + public Predicate> predicate() { return predicate; diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/predicate/FieldSet.java b/presto-spi/src/main/java/com/facebook/presto/spi/predicate/FieldSet.java new file mode 100644 index 000000000000..a7b8d5dffc76 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/predicate/FieldSet.java @@ -0,0 +1,45 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.predicate; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +public class FieldSet +{ + private final Set fields; + + @JsonCreator + public FieldSet(Set fields) + { + requireNonNull(fields, "fields is null"); + this.fields = fields; + } + + @JsonValue + public Set getFields() + { + return fields; + } + + @Override + public String toString() + { + return fields.toString(); + } +}