Skip to content

Commit

Permalink
Add support for overriding CheckConfig in forall (#278)
Browse files Browse the repository at this point in the history 
Addresses #265

- Fix small bug in stopping condition (7699d9c)
- Add support for overriding `CheckConfig` in `forall`
- Update README
  • Loading branch information
@dabd
dabd authored Apr 23, 2021
1 parent 6df89b2 commit e4a0dce
Show file tree
Hide file tree
Showing 3 changed files with 162 additions and 112 deletions.
13 changes: 13 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,10 @@ import weaver.scalacheck._
// Notice the Checkers mix-in
object ForallExamples extends SimpleIOSuite with Checkers {

// CheckConfig can be overridden at the test suite level
override def checkConfig: CheckConfig =
super.checkConfig.copy(perPropertyParallelism = 100)

test("Gen form") {
// Takes an explicit "Gen" instance. There is only a single
// version of this overload. If you want to pass several Gen instances
Expand All @@ -213,6 +217,15 @@ object ForallExamples extends SimpleIOSuite with Checkers {
}
}

test("foobar") {
// CheckConfig can be overridden locally
forall.withConfig(super.checkConfig.copy(perPropertyParallelism = 1,
initialSeed = Some(7L))) {
(x: Int) =>
expect(x > 0)
}
}

}
```

Expand Down
209 changes: 108 additions & 101 deletions modules/scalacheck/src/weaver/scalacheck/Checkers.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,116 +24,122 @@ trait Checkers {
// Configuration for property-based tests
def checkConfig: CheckConfig = CheckConfig.default

def forall[A1: Arbitrary: Show, B: PropF](f: A1 => B)(
implicit loc: SourceLocation): F[Expectations] =
forall(implicitly[Arbitrary[A1]].arbitrary)(liftProp[A1, B](f))

def forall[A1: Arbitrary: Show, A2: Arbitrary: Show, B: PropF](f: (
A1,
A2) => B)(
implicit loc: SourceLocation): F[Expectations] =
forall(implicitly[Arbitrary[(A1, A2)]].arbitrary)(liftProp(
f.tupled))

def forall[
A1: Arbitrary: Show,
A2: Arbitrary: Show,
A3: Arbitrary: Show,
B: PropF](
f: (A1, A2, A3) => B)(
implicit loc: SourceLocation): F[Expectations] = {
implicit val tuple3Show: Show[(A1, A2, A3)] = {
case (a1, a2, a3) => s"(${a1.show},${a2.show},${a3.show})"
class PartiallyAppliedForall(config: CheckConfig) {

def apply[A1: Arbitrary: Show, B: PropF](f: A1 => B)(
implicit loc: SourceLocation): F[Expectations] =
forall(implicitly[Arbitrary[A1]].arbitrary)(liftProp[A1, B](f))

def apply[A1: Arbitrary: Show, A2: Arbitrary: Show, B: PropF](f: (
A1,
A2) => B)(
implicit loc: SourceLocation): F[Expectations] =
forall(implicitly[Arbitrary[(A1, A2)]].arbitrary)(liftProp(
f.tupled))

def apply[
A1: Arbitrary: Show,
A2: Arbitrary: Show,
A3: Arbitrary: Show,
B: PropF](
f: (A1, A2, A3) => B)(
implicit loc: SourceLocation): F[Expectations] = {
implicit val tuple3Show: Show[(A1, A2, A3)] = {
case (a1, a2, a3) => s"(${a1.show},${a2.show},${a3.show})"
}
forall(implicitly[Arbitrary[(A1, A2, A3)]].arbitrary)(liftProp(
f.tupled))
}
forall(implicitly[Arbitrary[(A1, A2, A3)]].arbitrary)(liftProp(
f.tupled))
}

def forall[
A1: Arbitrary: Show,
A2: Arbitrary: Show,
A3: Arbitrary: Show,
A4: Arbitrary: Show,
B: PropF
](f: (A1, A2, A3, A4) => B)(
implicit loc: SourceLocation): F[Expectations] = {
implicit val tuple3Show: Show[(A1, A2, A3, A4)] = {
case (a1, a2, a3, a4) => s"(${a1.show},${a2.show},${a3.show},${a4.show})"
def apply[
A1: Arbitrary: Show,
A2: Arbitrary: Show,
A3: Arbitrary: Show,
A4: Arbitrary: Show,
B: PropF
](f: (A1, A2, A3, A4) => B)(
implicit loc: SourceLocation): F[Expectations] = {
implicit val tuple3Show: Show[(A1, A2, A3, A4)] = {
case (a1, a2, a3, a4) =>
s"(${a1.show},${a2.show},${a3.show},${a4.show})"
}
forall(implicitly[Arbitrary[(A1, A2, A3, A4)]].arbitrary)(
liftProp(f.tupled))
}
forall(implicitly[Arbitrary[(A1, A2, A3, A4)]].arbitrary)(
liftProp(f.tupled))
}

def forall[
A1: Arbitrary: Show,
A2: Arbitrary: Show,
A3: Arbitrary: Show,
A4: Arbitrary: Show,
A5: Arbitrary: Show,
B: PropF
](f: (A1, A2, A3, A4, A5) => B)(
implicit loc: SourceLocation): F[Expectations] = {
implicit val tuple3Show: Show[(A1, A2, A3, A4, A5)] = {
case (a1, a2, a3, a4, a5) =>
s"(${a1.show},${a2.show},${a3.show},${a4.show},${a5.show})"
def apply[
A1: Arbitrary: Show,
A2: Arbitrary: Show,
A3: Arbitrary: Show,
A4: Arbitrary: Show,
A5: Arbitrary: Show,
B: PropF
](f: (A1, A2, A3, A4, A5) => B)(
implicit loc: SourceLocation): F[Expectations] = {
implicit val tuple3Show: Show[(A1, A2, A3, A4, A5)] = {
case (a1, a2, a3, a4, a5) =>
s"(${a1.show},${a2.show},${a3.show},${a4.show},${a5.show})"
}
forall(implicitly[Arbitrary[(A1, A2, A3, A4, A5)]].arbitrary)(
liftProp(f.tupled))
}
forall(implicitly[Arbitrary[(A1, A2, A3, A4, A5)]].arbitrary)(
liftProp(f.tupled))
}

def forall[
A1: Arbitrary: Show,
A2: Arbitrary: Show,
A3: Arbitrary: Show,
A4: Arbitrary: Show,
A5: Arbitrary: Show,
A6: Arbitrary: Show,
B: PropF
](f: (A1, A2, A3, A4, A5, A6) => B)(
implicit loc: SourceLocation): F[Expectations] = {
implicit val tuple3Show: Show[(A1, A2, A3, A4, A5, A6)] = {
case (a1, a2, a3, a4, a5, a6) =>
s"(${a1.show},${a2.show},${a3.show},${a4.show},${a5.show},${a6.show})"
def apply[
A1: Arbitrary: Show,
A2: Arbitrary: Show,
A3: Arbitrary: Show,
A4: Arbitrary: Show,
A5: Arbitrary: Show,
A6: Arbitrary: Show,
B: PropF
](f: (A1, A2, A3, A4, A5, A6) => B)(
implicit loc: SourceLocation): F[Expectations] = {
implicit val tuple3Show: Show[(A1, A2, A3, A4, A5, A6)] = {
case (a1, a2, a3, a4, a5, a6) =>
s"(${a1.show},${a2.show},${a3.show},${a4.show},${a5.show},${a6.show})"
}
forall(implicitly[Arbitrary[(A1, A2, A3, A4, A5, A6)]].arbitrary)(
liftProp(f.tupled))
}
forall(implicitly[Arbitrary[(A1, A2, A3, A4, A5, A6)]].arbitrary)(
liftProp(f.tupled))
}

/** ScalaCheck test parameters instance. */
val numbers = fs2.Stream.iterate(1)(_ + 1)
def apply[A: Show, B: PropF](gen: Gen[A])(f: A => B)(
implicit loc: SourceLocation): F[Expectations] =
Ref[F].of(Status.start[A]).flatMap(forall_(gen, liftProp(f)))

def forall[A: Show, B: PropF](gen: Gen[A])(f: A => B)(
implicit loc: SourceLocation): F[Expectations] =
Ref[F].of(Status.start[A]).flatMap(forall_(gen, liftProp(f)))

private def forall_[A: Show](gen: Gen[A], f: A => F[Expectations])(
state: Ref[F, Status[A]])(
implicit loc: SourceLocation): F[Expectations] = {
paramStream
.parEvalMapUnordered(checkConfig.perPropertyParallelism) {
testOneTupled(gen, state, f)
}
.takeWhile(_.shouldContinue, takeFailure = true)
.takeRight(1) // getting the first error (which finishes the stream)
.compile
.last
.map { (x: Option[Status[A]]) =>
x match {
case Some(status) => status.endResult
case None => Expectations.Helpers.success
private def forall_[A: Show](gen: Gen[A], f: A => F[Expectations])(
state: Ref[F, Status[A]])(
implicit loc: SourceLocation): F[Expectations] = {
paramStream
.parEvalMapUnordered(config.perPropertyParallelism) {
testOneTupled(gen, state, f)
}
}
}
.takeWhile(_.shouldContinue, takeFailure = true)
.takeRight(1) // getting the first error (which finishes the stream)
.compile
.last
.map { (x: Option[Status[A]]) =>
x match {
case Some(status) => status.endResult
case None => Expectations.Helpers.success
}
}
}

private def paramStream: fs2.Stream[F, (Gen.Parameters, Seed)] = {
val initial = startSeed(
Gen.Parameters.default
.withSize(checkConfig.maximumGeneratorSize)
.withInitialSeed(checkConfig.initialSeed.map(Seed(_))))
private def paramStream: fs2.Stream[F, (Gen.Parameters, Seed)] = {
val initial = startSeed(
Gen.Parameters.default
.withSize(config.maximumGeneratorSize)
.withInitialSeed(config.initialSeed.map(Seed(_))))

fs2.Stream.iterate(initial) {
case (p, s) => (p, s.slide)
fs2.Stream.iterate(initial) {
case (p, s) => (p, s.slide)
}
}

}

object forall extends PartiallyAppliedForall(checkConfig) {
def withConfig(config: CheckConfig) = new PartiallyAppliedForall(config)
}

private def testOneTupled[T: Show](
Expand All @@ -154,7 +160,7 @@ trait Checkers {
.flatTap { (x: Option[(T, Expectations)]) =>
x match {
case Some((_, ex)) if ex.run.isValid => state.update(_.addSuccess)
case Some((t, ex)) => state.update(_.addFailure(t.show, ex))
case Some((t, ex)) => state.update(_.addFailure(t.show, seed, ex))
case None => state.update(_.addDiscard)
}
}
Expand All @@ -177,19 +183,20 @@ trait Checkers {
if (failure.isEmpty) copy(succeeded = succeeded + 1) else this
def addDiscard: Status[T] =
if (failure.isEmpty) copy(discarded = discarded + 1) else this
def addFailure(input: String, exp: Expectations): Status[T] =
def addFailure(input: String, seed: Seed, exp: Expectations): Status[T] =
if (failure.isEmpty) {
val ith = succeeded + discarded + 1
val failure = Expectations.Helpers
.failure(s"Property test failed on try $ith with input $input")
.failure(
s"Property test failed on try $ith with seed ${seed} and input $input")
.and(exp)
copy(failure = Some(failure))
} else this

def shouldStop =
failure.isDefined ||
succeeded >= checkConfig.minimumSuccessful ||
discarded >= checkConfig.maximumDiscardRatio
discarded >= checkConfig.maximumDiscarded

def shouldContinue = !shouldStop

Expand Down
52 changes: 41 additions & 11 deletions modules/scalacheck/test/src/weaver/scalacheck/PropertyDogFoodTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,20 @@ object PropertyDogFoodTest extends IOSuite {
// Go into software engineering they say
// Learn how to make amazing algorithms
// Build robust and deterministic software
val (attempt, value) =
if (ScalaCompat.isScala3)
("4", "-2147483648")
else
("2", "0")
val (attempt, value, seed) =
if (ScalaCompat.isScala3) {
("4",
"-2147483648",
"""Seed.fromBase64("AkTFK0oQzv-BOkf-rqnsdb_Etapzkj9gQD9rHj7UnKM=")""")
} else {
("2",
"0",
"""Seed.fromBase64("Nj62qCHF96VYEMGcD2OBlfmuyihbPQQhQLH9acYL5RA=")""")
}

val expectedMessage =
s"Property test failed on try $attempt with input $value"
s"Property test failed on try $attempt with seed $seed and input $value"

expect(log.contains(expectedMessage))
}
}
Expand All @@ -49,22 +55,38 @@ object PropertyDogFoodTest extends IOSuite {
}
}

// 5 checks with perPropertyParallelism = 1 sleeping 1 second each should take at least 5 seconds
test("Config can be overridden") { dogfood =>
for {
events <- dogfood.runSuite(Meta.ConfigOverrideChecks).map(_._2)
_ <- expect(events.size == 1).failFast
} yield {
expect(events.headOption.get.duration() >= 5000)
}
}
}

object Meta {
object ParallelChecks extends SimpleIOSuite with Checkers {
trait ParallelChecks extends SimpleIOSuite with Checkers {

override def checkConfig: CheckConfig =
super.checkConfig
.copy(perPropertyParallelism = 100, minimumSuccessful = 100)
def partiallyAppliedForall: PartiallyAppliedForall

test("sleeping forall") {
forall { (x: Int, y: Int) =>
partiallyAppliedForall { (x: Int, y: Int) =>
IO.sleep(1.second) *> IO(expect(x + y == y + x))
}
}
}

object ParallelChecks extends ParallelChecks {

override def partiallyAppliedForall: PartiallyAppliedForall = forall

override def checkConfig: CheckConfig =
super.checkConfig
.copy(perPropertyParallelism = 100, minimumSuccessful = 100)
}

object FailedChecks extends SimpleIOSuite with Checkers {

override def checkConfig: CheckConfig =
Expand All @@ -76,4 +98,12 @@ object Meta {
}
}
}

object ConfigOverrideChecks extends ParallelChecks {

override def partiallyAppliedForall: PartiallyAppliedForall =
forall.withConfig(super.checkConfig.copy(
perPropertyParallelism = 1,
minimumSuccessful = 5))
}
}

0 comments on commit e4a0dce

Please sign in to comment.