Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Non empty collection schemas (#717) #723

Merged
merged 4 commits into from
Aug 14, 2024
Merged

Non empty collection schemas (#717) #723

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
01a8804
Non empty collection schemas (#717)
987Nabil Aug 7, 2024
25b281d
Disable native multi threading
987Nabil Aug 13, 2024
61f5fd6
Merge branch 'main' into non-empty-collection
987Nabil Aug 14, 2024
5ceece9
Handle non-empty schemas
987Nabil Aug 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions build.sbt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,7 @@ inThisBuild(
ThisBuild / publishTo := sonatypePublishToBundle.value
scalacOptions ++= Seq("-scalajs")

addCommandAlias("prepare", "fix; fmt")
addCommandAlias("fmt", "all scalafmtSbt scalafmtAll")
addCommandAlias("fmt", "all scalafmtSbt scalafmtAll;fix")
addCommandAlias("fmtCheck", "all scalafmtSbtCheck scalafmtCheckAll")
addCommandAlias("fix", "scalafixAll")
addCommandAlias("fixCheck", "scalafixAll --check")
Expand Down
2 changes: 2 additions & 0 deletions tests/shared/src/test/scala/zio/schema/DynamicValueGen.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,9 @@ object DynamicValueGen {
case Schema.Enum22(_, case1, case2, case3, case4, case5, case6, case7, case8, case9, case10, case11, case12, case13, case14, case15, case16, case17, case18, case19, case20, case21, case22, _) => anyDynamicValueOfEnum(Chunk(case1, case2, case3, case4, case5, case6, case7, case8, case9, case10, case11, case12, case13, case14, case15, case16, case17, case18, case19, case20, case21, case22))
case Schema.EnumN(_, cases, _) => anyDynamicValueOfEnum(Chunk.fromIterable(cases.toSeq))
case Schema.Sequence(schema, _, _, _, _) => Gen.chunkOfBounded(0, 2)(anyDynamicValueOfSchema(schema)).map(DynamicValue.Sequence(_))
case Schema.NonEmptySequence(schema, _, _, _, _) => Gen.chunkOfBounded(1, 2)(anyDynamicValueOfSchema(schema)).map(DynamicValue.Sequence(_))
case Schema.Map(ks, vs, _) => Gen.chunkOfBounded(0, 2)(anyDynamicValueOfSchema(ks).zip(anyDynamicValueOfSchema(vs))).map(DynamicValue.Dictionary(_))
case Schema.NonEmptyMap(ks, vs, _) => Gen.chunkOfBounded(1, 2)(anyDynamicValueOfSchema(ks).zip(anyDynamicValueOfSchema(vs))).map(DynamicValue.Dictionary(_))
case Schema.Set(schema, _) => Gen.setOfBounded(0, 2)(anyDynamicValueOfSchema(schema)).map(DynamicValue.SetValue(_))
case Schema.Optional(schema, _) => Gen.oneOf(anyDynamicSomeValueOfSchema(schema), Gen.const(DynamicValue.NoneValue))
case Schema.Tuple2(left, right, _) => anyDynamicTupleValue(left, right)
Expand Down
30 changes: 26 additions & 4 deletions zio-schema-avro/src/main/scala/zio/schema/codec/AvroCodec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import org.apache.avro.io.{ DecoderFactory, EncoderFactory }
import org.apache.avro.util.Utf8
import org.apache.avro.{ Conversions, LogicalTypes, Schema => SchemaAvro }

import zio.prelude.NonEmptyMap
import zio.schema.{ Fallback, FieldSet, Schema, StandardType, TypeId }
import zio.stream.ZPipeline
import zio.{ Chunk, Unsafe, ZIO }
Expand Down Expand Up @@ -201,9 +202,20 @@ object AvroCodec {
case record: Schema.Record[_] => decodeRecord(raw, record).map(_.asInstanceOf[A])
case Schema.Sequence(element, f, _, _, _) =>
decodeSequence(raw, element.asInstanceOf[Schema[Any]]).map(f.asInstanceOf[Chunk[Any] => A])
case nes @ Schema.NonEmptySequence(element, _, _, _, _) =>
decodeSequence(raw, element.asInstanceOf[Schema[Any]]).map(nes.fromChunk.asInstanceOf[Chunk[Any] => A])
case Schema.Set(element, _) => decodeSequence(raw, element.asInstanceOf[Schema[Any]]).map(_.toSet.asInstanceOf[A])
case mapSchema: Schema.Map[_, _] =>
decodeMap(raw, mapSchema.asInstanceOf[Schema.Map[Any, Any]]).map(_.asInstanceOf[A])
case mapSchema: Schema.NonEmptyMap[_, _] =>
decodeMap(
raw,
Schema.Map(
mapSchema.keySchema.asInstanceOf[Schema[Any]],
mapSchema.valueSchema.asInstanceOf[Schema[Any]],
mapSchema.annotations
)
).map(mapSchema.asInstanceOf[Schema.NonEmptyMap[Any, Any]].fromMap(_).asInstanceOf[A])
case Schema.Transform(schema, f, _, _, _) =>
decodeValue(raw, schema).flatMap(
a => f(a).left.map(msg => DecodeError.MalformedFieldWithPath(Chunk.single("Error"), msg))
Expand Down Expand Up @@ -662,12 +674,22 @@ object AvroCodec {
c21,
c22
)
case Schema.GenericRecord(typeId, structure, _) => encodeGenericRecord(a, typeId, structure)
case Schema.Primitive(standardType, _) => encodePrimitive(a, standardType)
case Schema.Sequence(element, _, g, _, _) => encodeSequence(element, g(a))
case Schema.Set(element, _) => encodeSet(element, a)
case Schema.GenericRecord(typeId, structure, _) => encodeGenericRecord(a, typeId, structure)
case Schema.Primitive(standardType, _) => encodePrimitive(a, standardType)
case Schema.Sequence(element, _, g, _, _) => encodeSequence(element, g(a))
case Schema.NonEmptySequence(element, _, g, _, _) => encodeSequence(element, g(a))
case Schema.Set(element, _) => encodeSet(element, a)
case mapSchema: Schema.Map[_, _] =>
encodeMap(mapSchema.asInstanceOf[Schema.Map[Any, Any]], a.asInstanceOf[scala.collection.immutable.Map[Any, Any]])
case mapSchema: Schema.NonEmptyMap[_, _] =>
encodeMap(
Schema.Map(
mapSchema.keySchema.asInstanceOf[Schema[Any]],
mapSchema.valueSchema.asInstanceOf[Schema[Any]],
mapSchema.annotations
),
a.asInstanceOf[NonEmptyMap[Any, Any]].toMap
)
case Schema.Transform(schema, _, g, _, _) =>
g(a).map(encodeValue(_, schema)).getOrElse(throw new Exception("Transform failed."))
case Schema.Optional(schema, _) => encodeOption(schema, a)
Expand Down
28 changes: 22 additions & 6 deletions zio-schema-avro/src/main/scala/zio/schema/codec/AvroSchemaCodec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,12 +277,14 @@ object AvroSchemaCodec extends AvroSchemaCodec {

private def toAvroSchema(schema: Schema[_]): scala.util.Either[String, SchemaAvro] = {
schema match {
case e: Enum[_] => toAvroEnum(e)
case record: Record[_] => toAvroRecord(record)
case map: Schema.Map[_, _] => toAvroMap(map)
case seq: Schema.Sequence[_, _, _] => toAvroSchema(seq.elementSchema).map(SchemaAvro.createArray)
case set: Schema.Set[_] => toAvroSchema(set.elementSchema).map(SchemaAvro.createArray)
case Transform(codec, _, _, _, _) => toAvroSchema(codec)
case e: Enum[_] => toAvroEnum(e)
case record: Record[_] => toAvroRecord(record)
case map: Schema.Map[_, _] => toAvroMap(map)
case map: Schema.NonEmptyMap[_, _] => toAvroMap(map)
case seq: Schema.Sequence[_, _, _] => toAvroSchema(seq.elementSchema).map(SchemaAvro.createArray)
case seq: Schema.NonEmptySequence[_, _, _] => toAvroSchema(seq.elementSchema).map(SchemaAvro.createArray)
case set: Schema.Set[_] => toAvroSchema(set.elementSchema).map(SchemaAvro.createArray)
case Transform(codec, _, _, _, _) => toAvroSchema(codec)
case Primitive(standardType, _) =>
standardType match {
case StandardType.UnitType => Right(SchemaAvro.create(SchemaAvro.Type.NULL))
Expand Down Expand Up @@ -624,6 +626,18 @@ object AvroSchemaCodec extends AvroSchemaCodec {
toAvroSchema(tupleSchema).map(SchemaAvro.createArray)
}

private[codec] def toAvroMap(map: NonEmptyMap[_, _]): scala.util.Either[String, SchemaAvro] =
map.keySchema match {
case p: Schema.Primitive[_] if p.standardType == StandardType.StringType =>
toAvroSchema(map.valueSchema).map(SchemaAvro.createMap)
case _ =>
val tupleSchema = Schema
.Tuple2(map.keySchema, map.valueSchema)
.annotate(AvroAnnotations.name("Tuple"))
.annotate(AvroAnnotations.namespace("scala"))
toAvroSchema(tupleSchema).map(SchemaAvro.createArray)
}

private[codec] def toAvroDecimal(schema: Schema[_]): scala.util.Either[String, SchemaAvro] = {
val scale = schema.annotations.collectFirst { case AvroAnnotations.scale(s) => s }
.getOrElse(AvroAnnotations.scale().scale)
Expand Down Expand Up @@ -820,7 +834,9 @@ object AvroSchemaCodec extends AvroSchemaCodec {
case c: Dynamic => Right(c)
case c: GenericRecord => Right(c)
case c: Map[_, _] => Right(c)
case c: NonEmptyMap[_, _] => Right(c)
case c: Sequence[_, _, _] => Right(c)
case c: NonEmptySequence[_, _, _] => Right(c)
case c: Set[_] => Right(c)
case c: Fail[_] => Right(c)
case c: Lazy[_] => Right(c)
Expand Down
5 changes: 1 addition & 4 deletions zio-schema-avro/src/test/scala/zio/schema/codec/AvroSchemaCodecSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1918,10 +1918,7 @@ object AssertionHelper {
def recordFields(assertion: Assertion[Iterable[Schema.Field[_, _]]]): Assertion[Schema.Record[_]] =
Assertion.assertionRec[Schema.Record[_], Chunk[Field[_, _]]]("hasRecordField")(
assertion
) {
case r: Schema.Record[_] => Some(r.fields)
case _ => None
}
)((r: Schema.Record[_]) => Some(r.fields))

def hasSequenceElementSchema[A](assertion: Assertion[Schema[A]]): Assertion[Schema.Sequence[_, A, _]] =
Assertion.hasField("schemaA", _.elementSchema, assertion)
Expand Down
68 changes: 36 additions & 32 deletions zio-schema-bson/src/main/scala/zio/schema/codec/BsonSchemaCodec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -457,22 +457,24 @@ object BsonSchemaCodec {
//scalafmt: { maxColumn = 400, optIn.configStyleArguments = false }
private[codec] def schemaEncoder[A](schema: Schema[A]): BsonEncoder[A] =
schema match {
case Schema.Primitive(standardType, _) => primitiveCodec(standardType).encoder
case Schema.Sequence(schema, _, g, _, _) => chunkEncoder(schemaEncoder(schema)).contramap(g)
case Schema.Map(ks, vs, _) => mapEncoder(ks, vs)
case Schema.Set(s, _) => chunkEncoder(schemaEncoder(s)).contramap(m => Chunk.fromIterable(m))
case Schema.Transform(c, _, g, _, _) => transformEncoder(c, g)
case Schema.Tuple2(l, r, _) => tuple2Encoder(schemaEncoder(l), schemaEncoder(r))
case Schema.Optional(schema, _) => BsonEncoder.option(schemaEncoder(schema))
case Schema.Fail(_, _) => unitEncoder.contramap(_ => ())
case Schema.GenericRecord(_, structure, _) => genericRecordEncoder(structure.toChunk)
case Schema.Either(left, right, _) => eitherEncoder(schemaEncoder(left), schemaEncoder(right))
case Schema.Fallback(left, right, _, _) => fallbackEncoder(schemaEncoder(left), schemaEncoder(right))
case l @ Schema.Lazy(_) => schemaEncoder(l.schema)
case r: Schema.Record[A] => caseClassEncoder(r)
case e: Schema.Enum[A] => enumEncoder(e, e.cases)
case d @ Schema.Dynamic(_) => dynamicEncoder(d)
case null => throw new Exception(s"A captured schema is null, most likely due to wrong field initialization order")
case Schema.Primitive(standardType, _) => primitiveCodec(standardType).encoder
case Schema.Sequence(schema, _, g, _, _) => chunkEncoder(schemaEncoder(schema)).contramap(g)
case Schema.NonEmptySequence(schema, _, g, _, _) => chunkEncoder(schemaEncoder(schema)).contramap(g)
case Schema.Map(ks, vs, _) => mapEncoder(ks, vs)
case Schema.NonEmptyMap(ks, vs, _) => mapEncoder(ks, vs).contramap(_.toMap)
case Schema.Set(s, _) => chunkEncoder(schemaEncoder(s)).contramap(m => Chunk.fromIterable(m))
case Schema.Transform(c, _, g, _, _) => transformEncoder(c, g)
case Schema.Tuple2(l, r, _) => tuple2Encoder(schemaEncoder(l), schemaEncoder(r))
case Schema.Optional(schema, _) => BsonEncoder.option(schemaEncoder(schema))
case Schema.Fail(_, _) => unitEncoder.contramap(_ => ())
case Schema.GenericRecord(_, structure, _) => genericRecordEncoder(structure.toChunk)
case Schema.Either(left, right, _) => eitherEncoder(schemaEncoder(left), schemaEncoder(right))
case Schema.Fallback(left, right, _, _) => fallbackEncoder(schemaEncoder(left), schemaEncoder(right))
case l @ Schema.Lazy(_) => schemaEncoder(l.schema)
case r: Schema.Record[A] => caseClassEncoder(r)
case e: Schema.Enum[A] => enumEncoder(e, e.cases)
case d @ Schema.Dynamic(_) => dynamicEncoder(d)
case null => throw new Exception(s"A captured schema is null, most likely due to wrong field initialization order")
}
//scalafmt: { maxColumn = 120, optIn.configStyleArguments = true }

Expand Down Expand Up @@ -773,22 +775,24 @@ object BsonSchemaCodec {

//scalafmt: { maxColumn = 400, optIn.configStyleArguments = false }
private[codec] def schemaDecoder[A](schema: Schema[A]): BsonDecoder[A] = schema match {
case Schema.Primitive(standardType, _) => primitiveCodec(standardType).decoder
case Schema.Optional(codec, _) => BsonDecoder.option(schemaDecoder(codec))
case Schema.Tuple2(left, right, _) => tuple2Decoder(schemaDecoder(left), schemaDecoder(right))
case Schema.Transform(codec, f, _, _, _) => schemaDecoder(codec).mapOrFail(f)
case Schema.Sequence(codec, f, _, _, _) => chunkDecoder(schemaDecoder(codec)).map(f)
case Schema.Map(ks, vs, _) => mapDecoder(ks, vs)
case Schema.Set(s, _) => chunkDecoder(schemaDecoder(s)).map(entries => entries.toSet)
case Schema.Fail(message, _) => failDecoder(message)
case Schema.GenericRecord(_, structure, _) => recordDecoder(structure.toChunk)
case Schema.Either(left, right, _) => eitherDecoder(schemaDecoder(left), schemaDecoder(right))
case Schema.Fallback(left, right, _, _) => fallbackDecoder(schemaDecoder(left), schemaDecoder(right))
case l @ Schema.Lazy(_) => schemaDecoder(l.schema)
case s: Schema.Record[A] => caseClassDecoder(s)
case e: Schema.Enum[A] => enumDecoder(e)
case d @ Schema.Dynamic(_) => dynamicDecoder(d)
case null => throw new Exception(s"Missing a handler for decoding of schema $schema.")
case Schema.Primitive(standardType, _) => primitiveCodec(standardType).decoder
case Schema.Optional(codec, _) => BsonDecoder.option(schemaDecoder(codec))
case Schema.Tuple2(left, right, _) => tuple2Decoder(schemaDecoder(left), schemaDecoder(right))
case Schema.Transform(codec, f, _, _, _) => schemaDecoder(codec).mapOrFail(f)
case Schema.Sequence(codec, f, _, _, _) => chunkDecoder(schemaDecoder(codec)).map(f)
case s @ Schema.NonEmptySequence(codec, _, _, _, _) => chunkDecoder(schemaDecoder(codec)).map(s.fromChunk)
case Schema.Map(ks, vs, _) => mapDecoder(ks, vs)
case s @ Schema.NonEmptyMap(ks, vs, _) => mapDecoder(ks, vs).map(s.fromMap)
case Schema.Set(s, _) => chunkDecoder(schemaDecoder(s)).map(entries => entries.toSet)
case Schema.Fail(message, _) => failDecoder(message)
case Schema.GenericRecord(_, structure, _) => recordDecoder(structure.toChunk)
case Schema.Either(left, right, _) => eitherDecoder(schemaDecoder(left), schemaDecoder(right))
case Schema.Fallback(left, right, _, _) => fallbackDecoder(schemaDecoder(left), schemaDecoder(right))
case l @ Schema.Lazy(_) => schemaDecoder(l.schema)
case s: Schema.Record[A] => caseClassDecoder(s)
case e: Schema.Enum[A] => enumDecoder(e)
case d @ Schema.Dynamic(_) => dynamicDecoder(d)
case _ => throw new Exception(s"Missing a handler for decoding of schema $schema.")
}
//scalafmt: { maxColumn = 120, optIn.configStyleArguments = true }

Expand Down
4 changes: 2 additions & 2 deletions zio-schema-bson/src/test/scala/zio/schema/codec/BsonSchemaCodecSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,10 @@ object BsonSchemaCodecSpec extends ZIOSpecDefault {
implicit lazy val schema: Schema[Tree] = DeriveSchema.gen
implicit lazy val codec: BsonCodec[Tree] = BsonSchemaCodec.bsonCodec(schema)

private val genLeaf = Gen.int.map(Leaf)
private val genLeaf = Gen.int.map(Leaf.apply)

lazy val gen: Gen[Any, Tree] = Gen.sized { i =>
if (i >= 2) Gen.oneOf(genLeaf, Gen.suspend(gen.zipWith(gen)(Branch)).resize(i / 2))
if (i >= 2) Gen.oneOf(genLeaf, Gen.suspend(gen.zipWith(gen)(Branch.apply)).resize(i / 2))
else genLeaf
}
}
Expand Down
Loading
Loading