From 107776807a971084e117fcfbda815d292eefefc7 Mon Sep 17 00:00:00 2001 From: Charles O'Farrell Date: Thu, 4 Sep 2014 14:21:45 +1000 Subject: [PATCH] Revert "Merge pull request #337 from NICTA/topic/scalaz-7.1-upgrade" This reverts commit 3f5391af990ae080a464a521100c34a759735864, reversing changes made to 1a004fe889a232ca8e8be811dfa612ee7eeb0c87. Conflicts: project/dependencies.scala --- project/dependencies.scala | 2 +- .../scoobi/application/HadoopLogFactory.scala | 22 +++++++-------- .../com/nicta/scoobi/application/Levels.scala | 7 ++++- .../nicta/scoobi/application/ScoobiApp.scala | 5 ++-- .../nicta/scoobi/application/ScoobiArgs.scala | 2 +- .../scala/com/nicta/scoobi/core/DList.scala | 2 +- .../com/nicta/scoobi/core/Reduction.scala | 27 +++++++++---------- .../com/nicta/scoobi/lib/Relational.scala | 2 +- .../scoobi/acceptance/ScaldingSpec.scala | 2 -- .../scoobi/acceptance/SecondarySortSpec.scala | 2 -- .../scoobi/application/ScoobiAppSpec.scala | 2 +- 11 files changed, 37 insertions(+), 38 deletions(-) diff --git a/project/dependencies.scala b/project/dependencies.scala index 97ff15e36..ab59feeab 100644 --- a/project/dependencies.scala +++ b/project/dependencies.scala @@ -47,7 +47,7 @@ object dependencies { else if (version.contains("cdh5")) Seq("com.nicta" %% "scoobi-compatibility-cdh5" % "1.0.2") else Seq("com.nicta" %% "scoobi-compatibility-hadoop2" % "1.0.2") - def scalaz(scalazVersion: String = "7.1.0") = Seq( + def scalaz(scalazVersion: String = "7.0.6") = Seq( "org.scalaz" %% "scalaz-core" % scalazVersion, "org.scalaz" %% "scalaz-iteratee" % scalazVersion, "org.scalaz" %% "scalaz-concurrent" % scalazVersion, diff --git a/src/main/scala/com/nicta/scoobi/application/HadoopLogFactory.scala b/src/main/scala/com/nicta/scoobi/application/HadoopLogFactory.scala index 8dd3fe37b..88d815358 100644 --- a/src/main/scala/com/nicta/scoobi/application/HadoopLogFactory.scala +++ b/src/main/scala/com/nicta/scoobi/application/HadoopLogFactory.scala @@ -95,7 +95,7 @@ object HadoopLogFactory { def setLogFactory(name: String = classOf[HadoopLogFactory].getName, quiet: Boolean = false, showTimes: Boolean = false, - level: Level = INFO, + level: String = INFO, categories: String = ".*") { // release any previously set LogFactory for this class loader LogFactory.release(Thread.currentThread.getContextClassLoader) @@ -103,7 +103,7 @@ object HadoopLogFactory { setAttributes(quiet, showTimes, level, categories) } - def setAttributes(quiet: Boolean, showTimes: Boolean, level: Level, categories: String) { + def setAttributes(quiet: Boolean, showTimes: Boolean, level: String, categories: String) { setQuiet(quiet) setShowTimes(showTimes) setLogLevel(level) @@ -120,7 +120,7 @@ object HadoopLogFactory { def setShowTimes(showTimes: Boolean = false) { LogFactory.getFactory.setAttribute(SHOW_TIMES, showTimes) } - def setLogLevel(level: Level = INFO) { + def setLogLevel(level: String = INFO) { LogFactory.getFactory.setAttribute(LOG_LEVEL, level) } def setLogCategories(categories: String = ".*") { @@ -136,16 +136,16 @@ object HadoopLogFactory { FATAL -> SimpleLog.LOG_LEVEL_FATAL, OFF -> SimpleLog.LOG_LEVEL_OFF) - lazy val allLevels = levelsMappings.keys.map(_.level).toSet + lazy val allLevels = levelsMappings.keys.map(_.toString).toSet - lazy val ALL : Level = Level("ALL" ) - lazy val TRACE: Level = Level("TRACE") - lazy val INFO : Level = Level("INFO" ) - lazy val WARN : Level = Level("WARN" ) - lazy val ERROR: Level = Level("ERROR") - lazy val FATAL: Level = Level("FATAL") - lazy val OFF : Level = Level("OFF" ) + lazy val ALL : Level = level("ALL" ) + lazy val TRACE: Level = level("TRACE") + lazy val INFO : Level = level("INFO" ) + lazy val WARN : Level = level("WARN" ) + lazy val ERROR: Level = level("ERROR") + lazy val FATAL: Level = level("FATAL") + lazy val OFF : Level = level("OFF" ) } diff --git a/src/main/scala/com/nicta/scoobi/application/Levels.scala b/src/main/scala/com/nicta/scoobi/application/Levels.scala index c67773cb8..fb557aa20 100644 --- a/src/main/scala/com/nicta/scoobi/application/Levels.scala +++ b/src/main/scala/com/nicta/scoobi/application/Levels.scala @@ -16,7 +16,12 @@ package com.nicta.scoobi package application +import scalaz.@@ + object Levels { - case class Level(level: String) extends AnyVal + trait AsLevel + type Level = String @@ AsLevel + def level(l: String): Level = l.asInstanceOf[Level] + } diff --git a/src/main/scala/com/nicta/scoobi/application/ScoobiApp.scala b/src/main/scala/com/nicta/scoobi/application/ScoobiApp.scala index b74728a5f..279ccc989 100644 --- a/src/main/scala/com/nicta/scoobi/application/ScoobiApp.scala +++ b/src/main/scala/com/nicta/scoobi/application/ScoobiApp.scala @@ -16,7 +16,6 @@ package com.nicta.scoobi package application -import Levels._ import impl.io.FileSystems import impl.reflect.{ClasspathDiagnostics, Classes} import org.apache.commons.logging.LogFactory @@ -91,8 +90,8 @@ trait ScoobiApp extends ScoobiCommandLineArgs with ScoobiAppConfiguration with H // so that we know if configuration files must be read or not set(arguments) HadoopLogFactory.setLogFactory(classOf[HadoopLogFactory].getName, quiet, showTimes, level, categories) - configuration.set("mapred.map.child.log.level", level.level) - configuration.set("mapred.reduce.child.log.level", level.level) + configuration.set("mapred.map.child.log.level", level) + configuration.set("mapred.reduce.child.log.level", level) logger.debug("parsing the hadoop arguments "+ arguments.mkString(", ")) configuration.withHadoopArgs(arguments) { remainingArgs => diff --git a/src/main/scala/com/nicta/scoobi/application/ScoobiArgs.scala b/src/main/scala/com/nicta/scoobi/application/ScoobiArgs.scala index a5e743cde..ef3fa06c9 100644 --- a/src/main/scala/com/nicta/scoobi/application/ScoobiArgs.scala +++ b/src/main/scala/com/nicta/scoobi/application/ScoobiArgs.scala @@ -91,7 +91,7 @@ trait ScoobiUserArgs extends ScoobiArgs { private[scoobi] def extractLevel(args: Seq[String]): Level = - args.filter(a => allLevels contains a.toUpperCase).map(l => Level(l.toUpperCase)).headOption.getOrElse(INFO) + args.filter(a => allLevels contains a.toUpperCase).map(l => l.toUpperCase.asInstanceOf[Level]).headOption.getOrElse(INFO) /** * extract the categories as a regular expression from the scoobi arguments, once all the other argument names have been diff --git a/src/main/scala/com/nicta/scoobi/core/DList.scala b/src/main/scala/com/nicta/scoobi/core/DList.scala index 9ca940e48..3a4a0338e 100644 --- a/src/main/scala/com/nicta/scoobi/core/DList.scala +++ b/src/main/scala/com/nicta/scoobi/core/DList.scala @@ -298,7 +298,7 @@ trait DList[A] extends DataSinks with Persistent[Seq[A]] { // Oh, no. We have a key collision, so let's try randomize as much as we can else if (util.Random.nextBoolean()) scalaz.Ordering.LT else scalaz.Ordering.GT - override def groupCompare(a: Int, b: Int) = scalaz.Order.fromScalaOrdering[Int].order(a,b) + override def groupCompare(a: Int, b: Int) = implicitly[scalaz.Order[Int]].order(a,b) } groupWith(_ => util.Random.nextInt())(sgp).mapFlatten(_._2) } diff --git a/src/main/scala/com/nicta/scoobi/core/Reduction.scala b/src/main/scala/com/nicta/scoobi/core/Reduction.scala index 51389804c..1c0017430 100644 --- a/src/main/scala/com/nicta/scoobi/core/Reduction.scala +++ b/src/main/scala/com/nicta/scoobi/core/Reduction.scala @@ -18,7 +18,6 @@ package core import scalaz.{BijectionT, Semigroup, Kleisli, Cokleisli, Endo, Store, Order, Equal, Digit, Writer, Compose, NonEmptyList, EphemeralStream, Validation, \/-, -\/, \/, State, Ordering, Failure, Success, Apply} -import scalaz.Kleisli.kleisli import scalaz.syntax.equal._ import BijectionT._ import scala.reflect.ClassTag @@ -262,40 +261,40 @@ trait Reduction[A] { /** * Takes a reduction to a reduction on a unary function in an environment (Q). */ - def pointwiseK[Q[_], B](implicit A: Apply[Q]): Reduction[Kleisli[Q, B, A]] = - Reduction((g, h) => kleisli( + def pointwiseK[Q[+_], B](implicit A: Apply[Q]): Reduction[Kleisli[Q, B, A]] = + Reduction((g, h) => Kleisli( b => A.apply2(g(b), h(b))(reduce(_, _)) )) /** * Takes a reduction to a reduction on a binary function with an environment (Q) in return position. */ - def pointwise2K[Q[_], B, C](implicit A: Apply[Q]): Reduction[Kleisli[Q, (B, C), A]] = - Reduction((g, h) => kleisli { + def pointwise2K[Q[+_], B, C](implicit A: Apply[Q]): Reduction[Kleisli[Q, (B, C), A]] = + Reduction((g, h) => Kleisli { case (b, c) => A.apply2(g(b, c), h(b, c))(reduce(_, _)) }) /** * Takes a reduction to a reduction on a ternary function with an environment (Q) in return position. */ - def pointwise3K[Q[_], B, C, D](implicit A: Apply[Q]): Reduction[Kleisli[Q, (B, C, D), A]] = - Reduction((g, h) => kleisli { + def pointwise3K[Q[+_], B, C, D](implicit A: Apply[Q]): Reduction[Kleisli[Q, (B, C, D), A]] = + Reduction((g, h) => Kleisli { case (b, c, d) => A.apply2(g(b, c, d), h(b, c, d))(reduce(_, _)) }) /** * Takes a reduction to a reduction on an arity-4 function with an environment (Q) in return position. */ - def pointwise4K[Q[_], B, C, D, E](implicit A: Apply[Q]): Reduction[Kleisli[Q, (B, C, D, E), A]] = - Reduction((g, h) => kleisli { + def pointwise4K[Q[+_], B, C, D, E](implicit A: Apply[Q]): Reduction[Kleisli[Q, (B, C, D, E), A]] = + Reduction((g, h) => Kleisli { case (b, c, d, e) => A.apply2(g(b, c, d, e), h(b, c, d, e))(reduce(_, _)) }) /** * Takes a reduction to a reduction on an arity-5 function with an environment (Q) in return position. */ - def pointwise5K[Q[_], B, C, D, E, F](implicit A: Apply[Q]): Reduction[Kleisli[Q, (B, C, D, E, F), A]] = - Reduction((g, h) => kleisli { + def pointwise5K[Q[+_], B, C, D, E, F](implicit A: Apply[Q]): Reduction[Kleisli[Q, (B, C, D, E, F), A]] = + Reduction((g, h) => Kleisli { case (b, c, d, e, f) => A.apply2(g(b, c, d, e, f), h(b, c, d, e, f))(reduce(_, _)) }) @@ -365,7 +364,7 @@ trait Reduction[A] { /** * Lifts this reduction to a reduction with an environment. */ - def lift[F[_]](implicit A: Apply[F]): Reduction[F[A]] = + def lift[F[+_]](implicit A: Apply[F]): Reduction[F[A]] = Reduction((a1, a2) => A.apply2(a1, a2)(reduce(_, _))) @@ -373,13 +372,13 @@ trait Reduction[A] { * Takes a reduction to a reduction on state. */ def state[S]: Reduction[State[S, A]] = - lift[({type lam[a]=State[S, a]})#lam] + lift[({type lam[+a]=State[S, a]})#lam] /** * Takes a reduction to a reduction on writer. */ def writer[W: Semigroup]: Reduction[Writer[W, A]] = - lift[({type lam[a]=Writer[W, a]})#lam] + lift[({type lam[+a]=Writer[W, a]})#lam] /** * Maps a pair of functions on a reduction to produce a reduction. diff --git a/src/main/scala/com/nicta/scoobi/lib/Relational.scala b/src/main/scala/com/nicta/scoobi/lib/Relational.scala index cceb0418b..702751f19 100644 --- a/src/main/scala/com/nicta/scoobi/lib/Relational.scala +++ b/src/main/scala/com/nicta/scoobi/lib/Relational.scala @@ -288,7 +288,7 @@ object Relational { implicit val grouping = new Grouping[(K, Int)] { def groupCompare(a: (K, Int), b: (K, Int)) = - implicitly[Grouping[K]].groupCompare(a._1, b._1) |+| scalaz.Order.fromScalaOrdering[Int].order(a._2, b._2) + implicitly[Grouping[K]].groupCompare(a._1, b._1) |+| a._2 ?|? b._2 } Relational.join( diff --git a/src/test/scala/com/nicta/scoobi/acceptance/ScaldingSpec.scala b/src/test/scala/com/nicta/scoobi/acceptance/ScaldingSpec.scala index bafdeaf81..b9f5f9652 100644 --- a/src/test/scala/com/nicta/scoobi/acceptance/ScaldingSpec.scala +++ b/src/test/scala/com/nicta/scoobi/acceptance/ScaldingSpec.scala @@ -26,8 +26,6 @@ import Movies._ class ScaldingSpec extends NictaSimpleJobs { "Translation of http://blog.echen.me/2012/02/09/movie-recommendations-and-more-via-mapreduce-and-scalding to Scoobi" >> { implicit sc: SC => - implicit val IntOrdering = scalaz.Order.fromScalaOrdering[Int] - // very small list of ratings val ratings = DList( Rating(1, 1, 1), diff --git a/src/test/scala/com/nicta/scoobi/acceptance/SecondarySortSpec.scala b/src/test/scala/com/nicta/scoobi/acceptance/SecondarySortSpec.scala index eefa0fafc..3bf9a4dd1 100644 --- a/src/test/scala/com/nicta/scoobi/acceptance/SecondarySortSpec.scala +++ b/src/test/scala/com/nicta/scoobi/acceptance/SecondarySortSpec.scala @@ -55,8 +55,6 @@ object SecondarySort { import syntax.semigroup._ import syntax.order._ - implicit val StringOrdering = scalaz.Order.fromScalaOrdering[String] - val secondary: Grouping[(FirstName, LastName)] = new Grouping[(FirstName, LastName)] { override def partition(key: (FirstName, LastName), howManyReducers: Int): Int = implicitly[Grouping[FirstName]].partition(key._1, howManyReducers) diff --git a/src/test/scala/com/nicta/scoobi/application/ScoobiAppSpec.scala b/src/test/scala/com/nicta/scoobi/application/ScoobiAppSpec.scala index 3a7677ce5..4eba46528 100644 --- a/src/test/scala/com/nicta/scoobi/application/ScoobiAppSpec.scala +++ b/src/test/scala/com/nicta/scoobi/application/ScoobiAppSpec.scala @@ -41,7 +41,7 @@ class ScoobiAppSpec extends UnitSpecification with Tables { } "By defaults logs must be displayed, at the INFO level" >> { app.quiet aka "quiet" must beFalse - app.level.level must_== "INFO" + app.level must_== "INFO" } "If 'quiet' is passed on the command line then there must be no logs" >> { val app = new ScoobiApp { def run {} }