[FSTORE-1628] Branch 2.0.7 spark 3.5

deequ-scalastyle.xml

@@ -35,7 +35,7 @@
     </check>
 
     <check customId="argcount" level="error" class="org.scalastyle.scalariform.ParameterNumberChecker" enabled="true">
-        <parameters><parameter name="maxParameters"><![CDATA[10]]></parameter></parameters>
+        <parameters><parameter name="maxParameters"><![CDATA[20]]></parameter></parameters>
     </check>
 
     <check level="error" class="org.scalastyle.scalariform.NoFinalizeChecker" enabled="true"></check>

pom.xml

@@ -4,23 +4,12 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
-    <groupId>com.amazon.deequ</groupId>
-    <artifactId>deequ</artifactId>
+    <groupId>com.logicalclocks</groupId>
+    <artifactId>deequ_${scala.major.version}</artifactId>
+<!--    <groupId>com.amazon.deequ</groupId>-->
+<!--    <artifactId>deequ</artifactId>-->
     <version>2.0.7-spark-3.5</version>
 
-    <properties>
-        <maven.compiler.source>1.8</maven.compiler.source>
-        <maven.compiler.target>1.8</maven.compiler.target>
-        <encoding>UTF-8</encoding>
-
-        <scala.major.version>2.12</scala.major.version>
-        <scala.version>${scala.major.version}.10</scala.version>
-        <artifact.scala.version>${scala.major.version}</artifact.scala.version>
-        <scala-maven-plugin.version>4.8.1</scala-maven-plugin.version>
-
-        <spark.version>3.5.0</spark.version>
-    </properties>
-
     <name>deequ</name>
     <description>Deequ is a library built on top of Apache Spark for defining "unit tests for data",
         which measure data quality in large datasets.
@@ -67,6 +56,48 @@
         <url>https://github.com/awslabs/deequ</url>
     </scm>
 
+
+    <!-- awslabs/[email protected] does only have the following properties. There are not multiple profiles -->
+    <!-- <properties>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+        <encoding>UTF-8</encoding>
+
+        <scala.major.version>2.12</scala.major.version>
+        <scala.version>${scala.major.version}.10</scala.version>
+        <artifact.scala.version>${scala.major.version}</artifact.scala.version>
+        <scala-maven-plugin.version>4.8.1</scala-maven-plugin.version>
+
+        <spark.version>3.3.0</spark.version>
+    </properties> -->
+
+    <properties>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+        <encoding>UTF-8</encoding>
+
+        <!-- Scala -->
+        <scala.major.version>${scala-212.major.version}</scala.major.version>
+        <scala.version>${scala.major.version}.10</scala.version>
+        <scala-211.major.version>2.11</scala-211.major.version>
+        <scala-212.major.version>2.12</scala-212.major.version>
+        <artifact.scala.version>_scala-${scala.major.version}</artifact.scala.version>
+        <artifact.spark.version>_spark-${spark.version}</artifact.spark.version>
+        <scala-maven-plugin.version>4.8.1</scala-maven-plugin.version>
+
+        <!-- Spark -->
+        <spark.version>${spark-35.version}</spark.version>
+        <spark-22.version>2.2.2</spark-22.version>
+        <spark-23.version>2.3.2</spark-23.version>
+        <spark-24.version>2.4.2</spark-24.version>
+        <spark-30.version>3.0.0</spark-30.version>
+        <spark-31.version>3.1.1.0</spark-31.version>
+        <spark-33.version>3.3.0</spark-33.version>
+        <spark-35.version>3.5.0</spark-35.version>
+        <artifact.spark.version></artifact.spark.version>
+        <spark.scope>provided</spark.scope>
+    </properties>
+
     <dependencies>
         <dependency>
             <groupId>org.scala-lang</groupId>
@@ -86,12 +117,14 @@
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-core_${scala.major.version}</artifactId>
             <version>${spark.version}</version>
+            <scope>${spark.scope}</scope>
         </dependency>
 
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-sql_${scala.major.version}</artifactId>
             <version>${spark.version}</version>
+            <scope>${spark.scope}</scope>
         </dependency>
 
         <dependency>
@@ -422,6 +455,91 @@
                 </plugins>
             </build>
         </profile>
+
+        <!-- In logicalclocks/[email protected] we have multiple profiles. They are not anymore in awslabs/deequ-->
+
+        <!-- <profile>
+            <id>spark-2.2-scala-2.11</id>
+            <properties>
+                <spark.version>${spark-22.version}</spark.version>
+                <scala.major.version>${scala-211.major.version}</scala.major.version>
+                <scala.version>${scala.major.version}.10</scala.version>
+                <artifact.scala.version>_scala-${scala.major.version}</artifact.scala.version>
+                <artifact.spark.version>_spark-${spark.version}</artifact.spark.version>
+            </properties>
+        </profile>
+        <profile>
+            <id>spark-2.3-scala-2.11</id>
+            <properties>
+                <spark.version>${spark-23.version}</spark.version>
+                <scala.major.version>${scala-211.major.version}</scala.major.version>
+                <scala.version>${scala.major.version}.10</scala.version>
+                <artifact.scala.version>_scala-${scala.major.version}</artifact.scala.version>
+                <artifact.spark.version>_spark-${spark.version}</artifact.spark.version>
+            </properties>
+        </profile>
+        <profile>
+            <id>spark-2.4-scala-2.11</id>
+            <properties>
+                <spark.version>${spark-24.version}</spark.version>
+                <scala.major.version>${scala-211.major.version}</scala.major.version>
+                <scala.version>${scala.major.version}.10</scala.version>
+                <artifact.scala.version>_scala-${scala.major.version}</artifact.scala.version>
+                <artifact.spark.version>_spark-${spark.version}</artifact.spark.version>
+            </properties>
+        </profile>
+        <profile>
+            <id>spark-3.0-scala-2.12</id>
+            <properties>
+                <spark.version>${spark-30.version}</spark.version>
+                <scala.major.version>${scala-212.major.version}</scala.major.version>
+                <scala.version>${scala.major.version}.10</scala.version>
+                <artifact.scala.version>_scala-${scala.major.version}</artifact.scala.version>
+                <artifact.spark.version>_spark-${spark.version}</artifact.spark.version>
+            </properties>
+        </profile>
+        <profile>
+            <id>spark-3.1-scala-2.12</id>
+            <properties>
+                <spark.version>${spark-31.version}</spark.version>
+                <scala.major.version>${scala-212.major.version}</scala.major.version>
+                <scala.version>${scala.major.version}.10</scala.version>
+                <artifact.scala.version>_scala-${scala.major.version}</artifact.scala.version>
+                <artifact.spark.version>_spark-${spark.version}</artifact.spark.version>
+            </properties>
+        </profile>
+        <profile>
+            <id>spark-3.3-scala-2.12</id>
+            <properties>
+                <spark.version>${spark-330.version}</spark.version>
+                <scala.major.version>${scala-212.major.version}</scala.major.version>
+                <scala.version>${scala.major.version}.10</scala.version>
+                <artifact.scala.version>_scala-${scala.major.version}</artifact.scala.version>
+                <artifact.spark.version>_spark-${spark.version}</artifact.spark.version>
+            </properties>
+        </profile> -->
     </profiles>
 
+    <repositories>
+        <repository>
+            <id>Hops</id>
+            <name>Hops Repo</name>
+            <url>https://archiva.hops.works/repository/Hops/</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
+
+    <distributionManagement>
+        <repository>
+            <id>Hops</id>
+            <name>Hops Repo</name>
+            <url>https://archiva.hops.works/repository/Hops/</url>
+        </repository>
+    </distributionManagement>
+
 </project>

src/main/scala/com/amazon/deequ/analyzers/Completeness.scala

@@ -16,7 +16,7 @@
 
 package com.amazon.deequ.analyzers
 
-import com.amazon.deequ.analyzers.Preconditions.{hasColumn, isNotNested}
+import com.amazon.deequ.analyzers.Preconditions.{hasColumn}
 import org.apache.spark.sql.functions.sum
 import org.apache.spark.sql.types.{IntegerType, StructType}
 import Analyzers._
@@ -45,7 +45,7 @@ case class Completeness(column: String, where: Option[String] = None,
   }
 
   override protected def additionalPreconditions(): Seq[StructType => Unit] = {
-    hasColumn(column) :: isNotNested(column) :: Nil
+    hasColumn(column) :: Nil
   }
 
   override def filterCondition: Option[String] = where

src/main/scala/com/amazon/deequ/analyzers/DataType.scala

@@ -35,6 +35,7 @@ object DataTypeInstances extends Enumeration {
   val Integral: Value = Value(2)
   val Boolean: Value = Value(3)
   val String: Value = Value(4)
+  val Decimal: Value = Value(5)
 }
 
 case class DataTypeHistogram(

src/main/scala/com/amazon/deequ/analyzers/Histogram.scala

@@ -134,7 +134,7 @@ case class Histogram(
 }
 
 object Histogram {
-  val NullFieldReplacement = "NullValue"
+  val NullFieldReplacement = "-null-"
   val MaximumAllowedDetailBins = 1000
   val count_function = "count"
   val sum_function = "sum"

src/main/scala/com/amazon/deequ/analyzers/catalyst/DeequFunctions.scala

@@ -47,7 +47,7 @@ object DeequFunctions {
 
   /** Standard deviation with state */
   def stateful_stddev_pop(column: Column): Column = withAggregateFunction {
-    StatefulStdDevPop(column.expr)
+    StatefulStdDevPop(column.expr, true)
   }
 
   /** Approximate number of distinct values with state via HLL's */

src/main/scala/com/amazon/deequ/analyzers/runners/KLLRunner.scala

@@ -18,7 +18,7 @@ package com.amazon.deequ.analyzers.runners
 
 import com.amazon.deequ.analyzers.{Analyzer, KLLParameters, KLLSketch, KLLState, QuantileNonSample, State, StateLoader, StatePersister}
 import com.amazon.deequ.metrics.Metric
-import org.apache.spark.sql.types.{ByteType, DoubleType, FloatType, IntegerType, LongType, ShortType, StructType}
+import org.apache.spark.sql.types.{ByteType, DecimalType, DoubleType, FloatType, IntegerType, LongType, ShortType, StructType}
 import org.apache.spark.sql.{DataFrame, Row}
 
 @SerialVersionUID(1L)
@@ -84,6 +84,13 @@ class FloatQuantileNonSample(sketchSize: Int, shrinkingFactor: Double)
   override def itemAsDouble(item: Any): Double = item.asInstanceOf[Float].toDouble
 }
 
+@SerialVersionUID(1L)
+class DecimalQuantileNonSample(sketchSize: Int, shrinkingFactor: Double)
+  extends UntypedQuantileNonSample(sketchSize, shrinkingFactor) with Serializable {
+  override def itemAsDouble(item: Any): Double = item.asInstanceOf[java.math.BigDecimal]
+    .doubleValue()
+}
+
 object KLLRunner {
 
   def computeKLLSketchesInExtraPass(
@@ -139,6 +146,7 @@ object KLLRunner {
         case ShortType => new ShortQuantileNonSample(sketchSize, shrinkingFactor)
         case IntegerType => new IntQuantileNonSample(sketchSize, shrinkingFactor)
         case LongType => new LongQuantileNonSample(sketchSize, shrinkingFactor)
+        case DecimalType() => new DecimalQuantileNonSample(sketchSize, shrinkingFactor)
         // TODO at the moment, we will throw exceptions for Decimals
         case _ => throw new IllegalArgumentException(s"Cannot handle ${schema(column).dataType}")
       }

src/main/scala/com/amazon/deequ/examples/DataProfilingExample.scala

@@ -43,6 +43,7 @@ private[examples] object DataProfilingExample extends App {
        any shuffles. */
     val result = ColumnProfilerRunner()
       .onData(rawData)
+      .nonOptimized()
       .run()
 
     /* We get a profile for each column which allows to inspect the completeness of the column,