diff --git a/README.md b/README.md index 531440e8..eaccf557 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,10 @@ import weaver.scalacheck._ // Notice the Checkers mix-in object ForallExamples extends SimpleIOSuite with Checkers { + // CheckConfig can be overridden at the test suite level + override def checkConfig: CheckConfig = + super.checkConfig.copy(perPropertyParallelism = 100) + test("Gen form") { // Takes an explicit "Gen" instance. There is only a single // version of this overload. If you want to pass several Gen instances @@ -213,6 +217,15 @@ object ForallExamples extends SimpleIOSuite with Checkers { } } + test("foobar") { + // CheckConfig can be overridden locally + forall.withConfig(super.checkConfig.copy(perPropertyParallelism = 1, + initialSeed = Some(7L))) { + (x: Int) => + expect(x > 0) + } + } + } ``` diff --git a/modules/scalacheck/src/weaver/scalacheck/Checkers.scala b/modules/scalacheck/src/weaver/scalacheck/Checkers.scala index 11d6dc12..a6d31e02 100644 --- a/modules/scalacheck/src/weaver/scalacheck/Checkers.scala +++ b/modules/scalacheck/src/weaver/scalacheck/Checkers.scala @@ -24,116 +24,122 @@ trait Checkers { // Configuration for property-based tests def checkConfig: CheckConfig = CheckConfig.default - def forall[A1: Arbitrary: Show, B: PropF](f: A1 => B)( - implicit loc: SourceLocation): F[Expectations] = - forall(implicitly[Arbitrary[A1]].arbitrary)(liftProp[A1, B](f)) - - def forall[A1: Arbitrary: Show, A2: Arbitrary: Show, B: PropF](f: ( - A1, - A2) => B)( - implicit loc: SourceLocation): F[Expectations] = - forall(implicitly[Arbitrary[(A1, A2)]].arbitrary)(liftProp( - f.tupled)) - - def forall[ - A1: Arbitrary: Show, - A2: Arbitrary: Show, - A3: Arbitrary: Show, - B: PropF]( - f: (A1, A2, A3) => B)( - implicit loc: SourceLocation): F[Expectations] = { - implicit val tuple3Show: Show[(A1, A2, A3)] = { - case (a1, a2, a3) => s"(${a1.show},${a2.show},${a3.show})" + class PartiallyAppliedForall(config: CheckConfig) { + + def apply[A1: Arbitrary: Show, B: PropF](f: A1 => B)( + implicit loc: SourceLocation): F[Expectations] = + forall(implicitly[Arbitrary[A1]].arbitrary)(liftProp[A1, B](f)) + + def apply[A1: Arbitrary: Show, A2: Arbitrary: Show, B: PropF](f: ( + A1, + A2) => B)( + implicit loc: SourceLocation): F[Expectations] = + forall(implicitly[Arbitrary[(A1, A2)]].arbitrary)(liftProp( + f.tupled)) + + def apply[ + A1: Arbitrary: Show, + A2: Arbitrary: Show, + A3: Arbitrary: Show, + B: PropF]( + f: (A1, A2, A3) => B)( + implicit loc: SourceLocation): F[Expectations] = { + implicit val tuple3Show: Show[(A1, A2, A3)] = { + case (a1, a2, a3) => s"(${a1.show},${a2.show},${a3.show})" + } + forall(implicitly[Arbitrary[(A1, A2, A3)]].arbitrary)(liftProp( + f.tupled)) } - forall(implicitly[Arbitrary[(A1, A2, A3)]].arbitrary)(liftProp( - f.tupled)) - } - def forall[ - A1: Arbitrary: Show, - A2: Arbitrary: Show, - A3: Arbitrary: Show, - A4: Arbitrary: Show, - B: PropF - ](f: (A1, A2, A3, A4) => B)( - implicit loc: SourceLocation): F[Expectations] = { - implicit val tuple3Show: Show[(A1, A2, A3, A4)] = { - case (a1, a2, a3, a4) => s"(${a1.show},${a2.show},${a3.show},${a4.show})" + def apply[ + A1: Arbitrary: Show, + A2: Arbitrary: Show, + A3: Arbitrary: Show, + A4: Arbitrary: Show, + B: PropF + ](f: (A1, A2, A3, A4) => B)( + implicit loc: SourceLocation): F[Expectations] = { + implicit val tuple3Show: Show[(A1, A2, A3, A4)] = { + case (a1, a2, a3, a4) => + s"(${a1.show},${a2.show},${a3.show},${a4.show})" + } + forall(implicitly[Arbitrary[(A1, A2, A3, A4)]].arbitrary)( + liftProp(f.tupled)) } - forall(implicitly[Arbitrary[(A1, A2, A3, A4)]].arbitrary)( - liftProp(f.tupled)) - } - def forall[ - A1: Arbitrary: Show, - A2: Arbitrary: Show, - A3: Arbitrary: Show, - A4: Arbitrary: Show, - A5: Arbitrary: Show, - B: PropF - ](f: (A1, A2, A3, A4, A5) => B)( - implicit loc: SourceLocation): F[Expectations] = { - implicit val tuple3Show: Show[(A1, A2, A3, A4, A5)] = { - case (a1, a2, a3, a4, a5) => - s"(${a1.show},${a2.show},${a3.show},${a4.show},${a5.show})" + def apply[ + A1: Arbitrary: Show, + A2: Arbitrary: Show, + A3: Arbitrary: Show, + A4: Arbitrary: Show, + A5: Arbitrary: Show, + B: PropF + ](f: (A1, A2, A3, A4, A5) => B)( + implicit loc: SourceLocation): F[Expectations] = { + implicit val tuple3Show: Show[(A1, A2, A3, A4, A5)] = { + case (a1, a2, a3, a4, a5) => + s"(${a1.show},${a2.show},${a3.show},${a4.show},${a5.show})" + } + forall(implicitly[Arbitrary[(A1, A2, A3, A4, A5)]].arbitrary)( + liftProp(f.tupled)) } - forall(implicitly[Arbitrary[(A1, A2, A3, A4, A5)]].arbitrary)( - liftProp(f.tupled)) - } - def forall[ - A1: Arbitrary: Show, - A2: Arbitrary: Show, - A3: Arbitrary: Show, - A4: Arbitrary: Show, - A5: Arbitrary: Show, - A6: Arbitrary: Show, - B: PropF - ](f: (A1, A2, A3, A4, A5, A6) => B)( - implicit loc: SourceLocation): F[Expectations] = { - implicit val tuple3Show: Show[(A1, A2, A3, A4, A5, A6)] = { - case (a1, a2, a3, a4, a5, a6) => - s"(${a1.show},${a2.show},${a3.show},${a4.show},${a5.show},${a6.show})" + def apply[ + A1: Arbitrary: Show, + A2: Arbitrary: Show, + A3: Arbitrary: Show, + A4: Arbitrary: Show, + A5: Arbitrary: Show, + A6: Arbitrary: Show, + B: PropF + ](f: (A1, A2, A3, A4, A5, A6) => B)( + implicit loc: SourceLocation): F[Expectations] = { + implicit val tuple3Show: Show[(A1, A2, A3, A4, A5, A6)] = { + case (a1, a2, a3, a4, a5, a6) => + s"(${a1.show},${a2.show},${a3.show},${a4.show},${a5.show},${a6.show})" + } + forall(implicitly[Arbitrary[(A1, A2, A3, A4, A5, A6)]].arbitrary)( + liftProp(f.tupled)) } - forall(implicitly[Arbitrary[(A1, A2, A3, A4, A5, A6)]].arbitrary)( - liftProp(f.tupled)) - } - /** ScalaCheck test parameters instance. */ - val numbers = fs2.Stream.iterate(1)(_ + 1) + def apply[A: Show, B: PropF](gen: Gen[A])(f: A => B)( + implicit loc: SourceLocation): F[Expectations] = + Ref[F].of(Status.start[A]).flatMap(forall_(gen, liftProp(f))) - def forall[A: Show, B: PropF](gen: Gen[A])(f: A => B)( - implicit loc: SourceLocation): F[Expectations] = - Ref[F].of(Status.start[A]).flatMap(forall_(gen, liftProp(f))) - - private def forall_[A: Show](gen: Gen[A], f: A => F[Expectations])( - state: Ref[F, Status[A]])( - implicit loc: SourceLocation): F[Expectations] = { - paramStream - .parEvalMapUnordered(checkConfig.perPropertyParallelism) { - testOneTupled(gen, state, f) - } - .takeWhile(_.shouldContinue, takeFailure = true) - .takeRight(1) // getting the first error (which finishes the stream) - .compile - .last - .map { (x: Option[Status[A]]) => - x match { - case Some(status) => status.endResult - case None => Expectations.Helpers.success + private def forall_[A: Show](gen: Gen[A], f: A => F[Expectations])( + state: Ref[F, Status[A]])( + implicit loc: SourceLocation): F[Expectations] = { + paramStream + .parEvalMapUnordered(config.perPropertyParallelism) { + testOneTupled(gen, state, f) } - } - } + .takeWhile(_.shouldContinue, takeFailure = true) + .takeRight(1) // getting the first error (which finishes the stream) + .compile + .last + .map { (x: Option[Status[A]]) => + x match { + case Some(status) => status.endResult + case None => Expectations.Helpers.success + } + } + } - private def paramStream: fs2.Stream[F, (Gen.Parameters, Seed)] = { - val initial = startSeed( - Gen.Parameters.default - .withSize(checkConfig.maximumGeneratorSize) - .withInitialSeed(checkConfig.initialSeed.map(Seed(_)))) + private def paramStream: fs2.Stream[F, (Gen.Parameters, Seed)] = { + val initial = startSeed( + Gen.Parameters.default + .withSize(config.maximumGeneratorSize) + .withInitialSeed(config.initialSeed.map(Seed(_)))) - fs2.Stream.iterate(initial) { - case (p, s) => (p, s.slide) + fs2.Stream.iterate(initial) { + case (p, s) => (p, s.slide) + } } + + } + + object forall extends PartiallyAppliedForall(checkConfig) { + def withConfig(config: CheckConfig) = new PartiallyAppliedForall(config) } private def testOneTupled[T: Show]( @@ -154,7 +160,7 @@ trait Checkers { .flatTap { (x: Option[(T, Expectations)]) => x match { case Some((_, ex)) if ex.run.isValid => state.update(_.addSuccess) - case Some((t, ex)) => state.update(_.addFailure(t.show, ex)) + case Some((t, ex)) => state.update(_.addFailure(t.show, seed, ex)) case None => state.update(_.addDiscard) } } @@ -177,11 +183,12 @@ trait Checkers { if (failure.isEmpty) copy(succeeded = succeeded + 1) else this def addDiscard: Status[T] = if (failure.isEmpty) copy(discarded = discarded + 1) else this - def addFailure(input: String, exp: Expectations): Status[T] = + def addFailure(input: String, seed: Seed, exp: Expectations): Status[T] = if (failure.isEmpty) { val ith = succeeded + discarded + 1 val failure = Expectations.Helpers - .failure(s"Property test failed on try $ith with input $input") + .failure( + s"Property test failed on try $ith with seed ${seed} and input $input") .and(exp) copy(failure = Some(failure)) } else this @@ -189,7 +196,7 @@ trait Checkers { def shouldStop = failure.isDefined || succeeded >= checkConfig.minimumSuccessful || - discarded >= checkConfig.maximumDiscardRatio + discarded >= checkConfig.maximumDiscarded def shouldContinue = !shouldStop diff --git a/modules/scalacheck/test/src/weaver/scalacheck/PropertyDogFoodTest.scala b/modules/scalacheck/test/src/weaver/scalacheck/PropertyDogFoodTest.scala index b315cae7..865a3ffd 100644 --- a/modules/scalacheck/test/src/weaver/scalacheck/PropertyDogFoodTest.scala +++ b/modules/scalacheck/test/src/weaver/scalacheck/PropertyDogFoodTest.scala @@ -26,14 +26,20 @@ object PropertyDogFoodTest extends IOSuite { // Go into software engineering they say // Learn how to make amazing algorithms // Build robust and deterministic software - val (attempt, value) = - if (ScalaCompat.isScala3) - ("4", "-2147483648") - else - ("2", "0") + val (attempt, value, seed) = + if (ScalaCompat.isScala3) { + ("4", + "-2147483648", + """Seed.fromBase64("AkTFK0oQzv-BOkf-rqnsdb_Etapzkj9gQD9rHj7UnKM=")""") + } else { + ("2", + "0", + """Seed.fromBase64("Nj62qCHF96VYEMGcD2OBlfmuyihbPQQhQLH9acYL5RA=")""") + } val expectedMessage = - s"Property test failed on try $attempt with input $value" + s"Property test failed on try $attempt with seed $seed and input $value" + expect(log.contains(expectedMessage)) } } @@ -49,22 +55,38 @@ object PropertyDogFoodTest extends IOSuite { } } + // 5 checks with perPropertyParallelism = 1 sleeping 1 second each should take at least 5 seconds + test("Config can be overridden") { dogfood => + for { + events <- dogfood.runSuite(Meta.ConfigOverrideChecks).map(_._2) + _ <- expect(events.size == 1).failFast + } yield { + expect(events.headOption.get.duration() >= 5000) + } + } } object Meta { - object ParallelChecks extends SimpleIOSuite with Checkers { + trait ParallelChecks extends SimpleIOSuite with Checkers { - override def checkConfig: CheckConfig = - super.checkConfig - .copy(perPropertyParallelism = 100, minimumSuccessful = 100) + def partiallyAppliedForall: PartiallyAppliedForall test("sleeping forall") { - forall { (x: Int, y: Int) => + partiallyAppliedForall { (x: Int, y: Int) => IO.sleep(1.second) *> IO(expect(x + y == y + x)) } } } + object ParallelChecks extends ParallelChecks { + + override def partiallyAppliedForall: PartiallyAppliedForall = forall + + override def checkConfig: CheckConfig = + super.checkConfig + .copy(perPropertyParallelism = 100, minimumSuccessful = 100) + } + object FailedChecks extends SimpleIOSuite with Checkers { override def checkConfig: CheckConfig = @@ -76,4 +98,12 @@ object Meta { } } } + + object ConfigOverrideChecks extends ParallelChecks { + + override def partiallyAppliedForall: PartiallyAppliedForall = + forall.withConfig(super.checkConfig.copy( + perPropertyParallelism = 1, + minimumSuccessful = 5)) + } }