From 18e226e2ebcfec8ba890ad501787953a75798a6c Mon Sep 17 00:00:00 2001 From: Alexandre Cirilo <38657258+alxdrcirilo@users.noreply.github.com> Date: Wed, 10 Jul 2024 16:49:49 +0200 Subject: [PATCH] feat: render enum as (str, enum.Enum) (#637) (#652) * feat: render enum as (str, enum.Enum) (#637) * docs: fix typo in docs * fix(test): add missing newline in expected_result * feat: support (str, enum.Enum) and (enum.Enum) To use the @enum.nonmember decorator, we need Python >= 3.11. These changes aim at supporting (str, enum.Enum) and the use of @enum.nonmember for >= 3.11. But also to support >=3.8,<3.11 we omit @enum.nonmembmer and only use (enum.Enum). * docs: support >=3.11 (str, enum.Enum) and >=3.8,<3.11 (enum.Enum) --------- Co-authored-by: Marcos Schroh <2828842+marcosschroh@users.noreply.github.com> --- README.md | 2 +- dataclasses_avroschema/fields/fields.py | 10 +- .../model_generator/lang/python/base.py | 18 +- .../model_generator/lang/python/templates.py | 10 +- docs/case.md | 190 ++++++++----- docs/complex_types.md | 249 +++++++++++++----- docs/fields_specification.md | 202 +++++++++----- docs/index.md | 209 ++++++++++----- docs/migration_guide.md | 85 ++++-- docs/model_generator.md | 119 ++++++--- tests/conftest.py | 2 +- tests/model_generator/test_model_generator.py | 34 ++- .../test_model_pydantic_generator.py | 1 + 13 files changed, 780 insertions(+), 351 deletions(-) diff --git a/README.md b/README.md index 365e08b4..6688e868 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ import typing from dataclasses_avroschema import AvroModel, types -class FavoriteColor(enum.Enum): +class FavoriteColor(str, enum.Enum): BLUE = "BLUE" YELLOW = "YELLOW" GREEN = "GREEN" diff --git a/dataclasses_avroschema/fields/fields.py b/dataclasses_avroschema/fields/fields.py index 07c5ede5..07b39383 100644 --- a/dataclasses_avroschema/fields/fields.py +++ b/dataclasses_avroschema/fields/fields.py @@ -455,15 +455,19 @@ def __post_init__(self) -> None: def _get_meta_class_attributes(self) -> typing.Dict[str, typing.Any]: # get Enum members members = self.type.__members__ - meta = members.get("Meta") + meta = members.get("Meta") or getattr(self.type, "Meta", None) + doc: typing.Optional[str] = self.type.__doc__ # On python < 3.11 Enums have a default documentation so we remove it if version.PY_VERSION < (3, 11) and doc == "An enumeration.": doc = None - if meta is not None: - meta = meta.value + try: + if meta is not None: + meta = meta.value + except AttributeError: + pass metadata = utils.FieldMetadata.create(meta) if doc is not None: diff --git a/dataclasses_avroschema/model_generator/lang/python/base.py b/dataclasses_avroschema/model_generator/lang/python/base.py index 17a05942..1f64a1b8 100644 --- a/dataclasses_avroschema/model_generator/lang/python/base.py +++ b/dataclasses_avroschema/model_generator/lang/python/base.py @@ -88,7 +88,7 @@ def render_extras(self) -> str: return "".join([extra for extra in self.extras]) def render_metaclass( - self, *, schema: JsonDict, field_order: typing.Optional[typing.List[str]] = None + self, *, schema: JsonDict, field_order: typing.Optional[typing.List[str]] = None, decorator: str = "" ) -> typing.Optional[str]: """ Render Class Meta that contains the schema matadata @@ -116,12 +116,14 @@ def render_metaclass( if properties: # some formating to remove identation at the end of the Class Meta to make it more compatible with black - return ( - self.field_identation.join( - [line for line in templates.metaclass_template.safe_substitute(properties=properties).split("\n")] - ).rstrip(self.field_identation) - + "\n" - ) + return self.field_identation.join( + [ + line + for line in templates.metaclass_template.safe_substitute( + properties=properties, decorator=decorator + ).split("\n") + ] + ).rstrip(self.field_identation) return None def render_docstring(self, *, docstring: typing.Optional[str]) -> str: @@ -427,7 +429,7 @@ def parse_enum(self, field: JsonDict) -> str: docstring = self.render_docstring(docstring=field.get("doc")) enum_class = templates.enum_template.safe_substitute(name=enum_name, symbols=symbols_repr, docstring=docstring) - metaclass = self.render_metaclass(schema=field) + metaclass = self.render_metaclass(schema=field, decorator=templates.METACLASS_DECORATOR) if metaclass: enum_class += metaclass diff --git a/dataclasses_avroschema/model_generator/lang/python/templates.py b/dataclasses_avroschema/model_generator/lang/python/templates.py index e6c61123..b0d318ed 100644 --- a/dataclasses_avroschema/model_generator/lang/python/templates.py +++ b/dataclasses_avroschema/model_generator/lang/python/templates.py @@ -1,9 +1,13 @@ +import sys from string import Template +PYTHON_VERSION_GE_311 = sys.version_info.major == 3 and sys.version_info.minor >= 11 + FIELD_TYPE_TEMPLATE = "$name: $type" METACLASS_FIELD_TEMPLATE = '$name = "$value"' METACLASS_ALIAS_FIELD = "$name = $value" METACLASS_SCHEMA_FIELD = "$name = '$schema'" +METACLASS_DECORATOR = "@enum.nonmember" if PYTHON_VERSION_GE_311 else "" FIELD_DEFAULT_TEMPLATE = " = $default" OPTIONAL_TEMPLATE = "typing.Optional[$type]" UNION_TEMPLATE = "typing.Union[$type]" @@ -23,9 +27,10 @@ DECIMAL_TYPE_TEMPLATE = "types.condecimal(max_digits=$precision, decimal_places=$scale)" ENUM_SYMBOL_TEMPLATE = "$key = $value" -ENUM_TEMPLATE = """ +ENUM_PYTHON_VERSION = "str, enum.Enum" if PYTHON_VERSION_GE_311 else "enum.Enum" +ENUM_TEMPLATE = f""" -class $name(enum.Enum):$docstring +class $name({ENUM_PYTHON_VERSION}):$docstring $symbols """ @@ -38,6 +43,7 @@ class $name($base_class):$docstring INSTANCE_TEMPLATE = "$type($properties)" METACLASS_TEMPLATE = """ +$decorator class Meta: $properties """ diff --git a/docs/case.md b/docs/case.md index d3639088..878bad30 100644 --- a/docs/case.md +++ b/docs/case.md @@ -3,68 +3,134 @@ Sometimes we use `avro schemas` with different sources (some written in Scala, some in Python, etc). With the `case` you can generate your schemas according to your programming language convention: -```python title="Example with CAPITALCASE" -import typing -import dataclasses -import enum - -from dataclasses_avroschema import AvroModel, case, types - - -# New enum!! -class FavoriteColor(enum.Enum): - BLUE = "BLUE" - YELLOW = "YELLOW" - GREEN = "GREEN" - - -@dataclasses.dataclass -class UserAdvance(AvroModel): - name: str - age: int - pets: typing.List[str] - accounts: typing.Dict[str, int] - favorite_colors: FavoriteColor - has_car: bool = False - country: str = "Argentina" - address: str = None - md5: types.Fixed = types.Fixed(16) - - class Meta: - schema_doc = False - - -UserAdvance.avro_schema(case_type=case.CAPITALCASE) -``` - -resulting in - -```json -{ - "type": "record", - "name": "UserAdvance", - "fields": [ - {"name": "Name", "type": "string"}, - {"name": "Age", "type": "long"}, - {"name": "Pets", "type": { - "type": "array", "items": "string", "name": "Pet" - } - }, - {"name": "Accounts", "type": { - "type": "map", "values": "long", "name": "Account" - } - }, - {"name": "Has_car", "type": "boolean", "default": false}, - {"name": "Favorite_colors", "type": { - "type": "enum", "name": "FavoriteColor", "symbols": ["BLUE", "YELLOW", "GREEN"] - } - }, - {"name": "Country", "type": "string", "default": "Argentina"}, - {"name": "Address", "type": ["null", "string"], "default": null}, - {"name": "Md5", "type": {"type": "fixed", "name": "Md5", "size": 16}} - ] -}' -``` +=== "python <= 3.10" + + ```python title="Example with CAPITALCASE" + import typing + import dataclasses + import enum + + from dataclasses_avroschema import AvroModel, case, types + + + class FavoriteColor(enum.Enum): + BLUE = "BLUE" + YELLOW = "YELLOW" + GREEN = "GREEN" + + + @dataclasses.dataclass + class UserAdvance(AvroModel): + name: str + age: int + pets: typing.List[str] + accounts: typing.Dict[str, int] + favorite_colors: FavoriteColor + has_car: bool = False + country: str = "Argentina" + address: str = None + md5: types.Fixed = types.Fixed(16) + + class Meta: + schema_doc = False + + + UserAdvance.avro_schema(case_type=case.CAPITALCASE) + ``` + + resulting in + + ```json + { + "type": "record", + "name": "UserAdvance", + "fields": [ + {"name": "Name", "type": "string"}, + {"name": "Age", "type": "long"}, + {"name": "Pets", "type": { + "type": "array", "items": "string", "name": "Pet" + } + }, + {"name": "Accounts", "type": { + "type": "map", "values": "long", "name": "Account" + } + }, + {"name": "Has_car", "type": "boolean", "default": false}, + {"name": "Favorite_colors", "type": { + "type": "enum", "name": "FavoriteColor", "symbols": ["BLUE", "YELLOW", "GREEN"] + } + }, + {"name": "Country", "type": "string", "default": "Argentina"}, + {"name": "Address", "type": ["null", "string"], "default": null}, + {"name": "Md5", "type": {"type": "fixed", "name": "Md5", "size": 16}} + ] + }' + ``` + +=== "python >= 3.11" + + ```python title="Example with CAPITALCASE" + import typing + import dataclasses + import enum + + from dataclasses_avroschema import AvroModel, case, types + + + # New enum!! + class FavoriteColor(str, enum.Enum): + BLUE = "BLUE" + YELLOW = "YELLOW" + GREEN = "GREEN" + + + @dataclasses.dataclass + class UserAdvance(AvroModel): + name: str + age: int + pets: typing.List[str] + accounts: typing.Dict[str, int] + favorite_colors: FavoriteColor + has_car: bool = False + country: str = "Argentina" + address: str = None + md5: types.Fixed = types.Fixed(16) + + class Meta: + schema_doc = False + + + UserAdvance.avro_schema(case_type=case.CAPITALCASE) + ``` + + resulting in + + ```json + { + "type": "record", + "name": "UserAdvance", + "fields": [ + {"name": "Name", "type": "string"}, + {"name": "Age", "type": "long"}, + {"name": "Pets", "type": { + "type": "array", "items": "string", "name": "Pet" + } + }, + {"name": "Accounts", "type": { + "type": "map", "values": "long", "name": "Account" + } + }, + {"name": "Has_car", "type": "boolean", "default": false}, + {"name": "Favorite_colors", "type": { + "type": "enum", "name": "FavoriteColor", "symbols": ["BLUE", "YELLOW", "GREEN"] + } + }, + {"name": "Country", "type": "string", "default": "Argentina"}, + {"name": "Address", "type": ["null", "string"], "default": null}, + {"name": "Md5", "type": {"type": "fixed", "name": "Md5", "size": 16}} + ] + }' + ``` *(This script is complete, it should run "as is")* diff --git a/docs/complex_types.md b/docs/complex_types.md index 04533146..17d59435 100644 --- a/docs/complex_types.md +++ b/docs/complex_types.md @@ -2,72 +2,146 @@ The following list represent the avro complex types mapped to python types: -| Avro Type | Python Type | -| ------------------ | ------------------------------------------------------------------ | -| enums | enum.Enum, typing.Literal[str] | -| arrays | typing.List, typing.Tuple, typing.Sequence, typing.MutableSequence | -| maps | typing.Dict, typing.Mapping, typing.MutableMapping | -| fixed | types.confixed | -| unions | typing.Union | -| unions with `null` | typing.Optional | -| records | Python Class | +=== "python <= 3.10" + + | Avro Type | Python Type | + | ------------------ | ------------------------------------------------------------------ | + | enums | enum.Enum, typing.Literal[str] | + | arrays | typing.List, typing.Tuple, typing.Sequence, typing.MutableSequence | + | maps | typing.Dict, typing.Mapping, typing.MutableMapping | + | fixed | types.confixed | + | unions | typing.Union | + | unions with `null` | typing.Optional | + | records | Python Class | + +=== "python >= 3.11" + + | Avro Type | Python Type | + | ------------------ | ------------------------------------------------------------------ | + | enums | str, enum.Enum, typing.Literal[str] | + | arrays | typing.List, typing.Tuple, typing.Sequence, typing.MutableSequence | + | maps | typing.Dict, typing.Mapping, typing.MutableMapping | + | fixed | types.confixed | + | unions | typing.Union | + | unions with `null` | typing.Optional | + | records | Python Class | ## Enums -```python title="Enum example" -import enum -import dataclasses +=== "python <= 3.10" -from dataclasses_avroschema import AvroModel + ```python title="Enum example" + import enum + import dataclasses + from dataclasses_avroschema import AvroModel -class FavoriteColor(enum.Enum): - BLUE = "Blue" - YELLOW = "Yellow" - GREEN = "Green" - - class Meta: - doc = "A favorite color" - namespace = "some.name.space" - aliases = ["Color", "My favorite color"] + class FavoriteColor(enum.Enum): + BLUE = "Blue" + YELLOW = "Yellow" + GREEN = "Green" + + class Meta: + doc = "A favorite color" + namespace = "some.name.space" + aliases = ["Color", "My favorite color"] -@dataclasses.dataclass -class User(AvroModel): - "An User" - favorite_color: FavoriteColor = FavoriteColor.BLUE + @dataclasses.dataclass + class User(AvroModel): + "An User" + favorite_color: FavoriteColor = FavoriteColor.BLUE -User.avro_schema() -'{ - "type": "record", - "name": "User", - "fields": - [ - { - "name": "favorite_color", - "type": - { - "type": "enum", - "name": "FavoriteColor", - "symbols": - [ - "Blue", - "Yellow", - "Green" - ], - "doc": "A favorite color", - "namespace": "some.name.space", - "aliases": - ["Color", "My favorite color"] - }, - "default": "Blue" - } - ], - "doc": "An User" -}' -``` + User.avro_schema() + + '{ + "type": "record", + "name": "User", + "fields": + [ + { + "name": "favorite_color", + "type": + { + "type": "enum", + "name": "FavoriteColor", + "symbols": + [ + "Blue", + "Yellow", + "Green" + ], + "doc": "A favorite color", + "namespace": "some.name.space", + "aliases": + ["Color", "My favorite color"] + }, + "default": "Blue" + } + ], + "doc": "An User" + }' + ``` + +=== "python >= 3.11" + + ```python title="Enum example" + import enum + import dataclasses + + from dataclasses_avroschema import AvroModel + + + class FavoriteColor(str, enum.Enum): + BLUE = "Blue" + YELLOW = "Yellow" + GREEN = "Green" + + @enum.nonmember + class Meta: + doc = "A favorite color" + namespace = "some.name.space" + aliases = ["Color", "My favorite color"] + + + @dataclasses.dataclass + class User(AvroModel): + "An User" + favorite_color: FavoriteColor = FavoriteColor.BLUE + + + User.avro_schema() + + '{ + "type": "record", + "name": "User", + "fields": + [ + { + "name": "favorite_color", + "type": + { + "type": "enum", + "name": "FavoriteColor", + "symbols": + [ + "Blue", + "Yellow", + "Green" + ], + "doc": "A favorite color", + "namespace": "some.name.space", + "aliases": + ["Color", "My favorite color"] + }, + "default": "Blue" + } + ], + "doc": "An User" + }' + ``` !!! info There are not restriction about `enum` names but is is highly recommended to use `pascalcase` @@ -131,31 +205,62 @@ For this behaviour the attribute `convert_literal_to_enum` must be set to `True` Sometimes we have cases where an `Enum` is used more than once with a particular class, for those cases the same `type` is used in order to generate a valid schema. It is a good practice but *NOT* neccesary to a define the `namespace` on the repeated `type`. -```python -import enum -import dataclasses -import typing +=== "python <= 3.10" -from dataclasses_avroschema import AvroModel + ```python + import enum + import dataclasses + import typing + from dataclasses_avroschema import AvroModel -class TripDistance(enum.Enum): - CLOSE = "Close" - FAR = "Far" - class Meta: - doc = "Distance of the trip" - namespace = "trip" + class TripDistance(enum.Enum): + CLOSE = "Close" + FAR = "Far" + class Meta: + doc = "Distance of the trip" + namespace = "trip" -@dataclasses.dataclass -class User(AvroModel): - trip_distance: TripDistance - optional_distance: typing.Optional[TripDistance] = None + + @dataclasses.dataclass + class User(AvroModel): + trip_distance: TripDistance + optional_distance: typing.Optional[TripDistance] = None -print(User.avro_schema()) -``` + print(User.avro_schema()) + ``` + +=== "python >= 3.11" + + ```python + import enum + import dataclasses + import typing + + from dataclasses_avroschema import AvroModel + + + class TripDistance(str, enum.Enum): + CLOSE = "Close" + FAR = "Far" + + @enum.nonmember + class Meta: + doc = "Distance of the trip" + namespace = "trip" + + + @dataclasses.dataclass + class User(AvroModel): + trip_distance: TripDistance + optional_distance: typing.Optional[TripDistance] = None + + + print(User.avro_schema()) + ``` resulting in diff --git a/docs/fields_specification.md b/docs/fields_specification.md index 5c56f872..25aa99f8 100644 --- a/docs/fields_specification.md +++ b/docs/fields_specification.md @@ -119,36 +119,71 @@ Language implementations must ignore unknown logical types when reading, and sho ### Avro Field and Python Types Summary -Python Type | Avro Type | Logical Type | -|-----------|-------------|--------------| -| str | string | do not apply | -| long | int | do not apply | -| bool | boolean | do not apply | -| double | float | do not apply | -| None | null | do not apply | -| bytes | bytes | do not apply | -| typing.List | array | do not apply | -| typing.Tuple | array | do not apply | -| typing.Sequence | array | do not apply | -| typing.MutableSequence | array | do not apply | -| typing.Dict | map | do not apply | -| typing.Mapping | map | do not apply | -| typing.MutableMapping | map | do not apply | -| types.Fixed | fixed | do not apply | -| enum.Enum | enum | do not apply | -| types.Int32 | int | do not apply | -| types.Float32 | float| do not apply | -| typing.Union| union | do not apply | -| typing.Optional| union (with `null`) | do not apply | -| Python class | record | do not apply | -| datetime.date | int | date | -| datetime.time | int | time-millis | -| types.TimeMicro | long | time-micros | -| datetime.datetime| long | timestamp-millis | -| types.DateTimeMicro| long | timestamp-micros | -| decimal.Decimal | bytes | decimal | -| uuid.uuid4 | string | uuid | -| uuid.UUID | string | uuid | +=== "python <= 3.10" + + Python Type | Avro Type | Logical Type | + |-----------|-------------|--------------| + | str | string | do not apply | + | long | int | do not apply | + | bool | boolean | do not apply | + | double | float | do not apply | + | None | null | do not apply | + | bytes | bytes | do not apply | + | typing.List | array | do not apply | + | typing.Tuple | array | do not apply | + | typing.Sequence | array | do not apply | + | typing.MutableSequence | array | do not apply | + | typing.Dict | map | do not apply | + | typing.Mapping | map | do not apply | + | typing.MutableMapping | map | do not apply | + | types.Fixed | fixed | do not apply | + | enum.Enum | enum | do not apply | + | types.Int32 | int | do not apply | + | types.Float32 | float| do not apply | + | typing.Union| union | do not apply | + | typing.Optional| union (with `null`) | do not apply | + | Python class | record | do not apply | + | datetime.date | int | date | + | datetime.time | int | time-millis | + | types.TimeMicro | long | time-micros | + | datetime.datetime| long | timestamp-millis | + | types.DateTimeMicro| long | timestamp-micros | + | decimal.Decimal | bytes | decimal | + | uuid.uuid4 | string | uuid | + | uuid.UUID | string | uuid | + +=== "python >= 3.11" + + Python Type | Avro Type | Logical Type | + |-----------|-------------|--------------| + | str | string | do not apply | + | long | int | do not apply | + | bool | boolean | do not apply | + | double | float | do not apply | + | None | null | do not apply | + | bytes | bytes | do not apply | + | typing.List | array | do not apply | + | typing.Tuple | array | do not apply | + | typing.Sequence | array | do not apply | + | typing.MutableSequence | array | do not apply | + | typing.Dict | map | do not apply | + | typing.Mapping | map | do not apply | + | typing.MutableMapping | map | do not apply | + | types.Fixed | fixed | do not apply | + | str, enum.Enum | enum | do not apply | + | types.Int32 | int | do not apply | + | types.Float32 | float| do not apply | + | typing.Union| union | do not apply | + | typing.Optional| union (with `null`) | do not apply | + | Python class | record | do not apply | + | datetime.date | int | date | + | datetime.time | int | time-millis | + | types.TimeMicro | long | time-micros | + | datetime.datetime| long | timestamp-millis | + | types.DateTimeMicro| long | timestamp-micros | + | decimal.Decimal | bytes | decimal | + | uuid.uuid4 | string | uuid | + | uuid.UUID | string | uuid | ## typing.Annotated @@ -210,46 +245,91 @@ resulting in Fields can be annotated with `typing.Literal` in accordance with [PEP 586](https://peps.python.org/pep-0586/). Note that a literal field with multiple arguments (i.e. of the form `typing.Literal[v1, v2, v3]`) is interpreted as a union of literals (i.e. `typing.Union[typing.Literal[v1], typing.Literal[v2], typing.Literal[v3]]`) in line with the PEP. -```python -import enum -import typing -from dataclasses import dataclass -from dataclasses_avroschema import AvroModel +=== "python <= 3.10" -class E(enum.Enum): - ONE = "one" + ```python + import enum + import typing + from dataclasses import dataclass + from dataclasses_avroschema import AvroModel -@dataclass -class T(AvroModel): - f: typing.Literal[None, 1, "1", True, b"1", E.ONE] + class E(enum.Enum): + ONE = "one" -print(T.avro_schema()) -""" -{ - "type": "record", - "name": "T", - "fields": [ + @dataclass + class T(AvroModel): + f: typing.Literal[None, 1, "1", True, b"1", E.ONE] + + print(T.avro_schema()) + """ { - "name": "f", - "type": [ - "null", - "long", - "string", - "boolean", - "bytes", + "type": "record", + "name": "T", + "fields": [ { - "type": "enum", - "name": "E", - "symbols": [ - "one" + "name": "f", + "type": [ + "null", + "long", + "string", + "boolean", + "bytes", + { + "type": "enum", + "name": "E", + "symbols": [ + "one" + ] + } ] } ] } - ] -} -""" -``` + """ + ``` + +=== "python >= 3.11" + + ```python + import enum + import typing + from dataclasses import dataclass + from dataclasses_avroschema import AvroModel + + class E(str, enum.Enum): + ONE = "one" + + @dataclass + class T(AvroModel): + f: typing.Literal[None, 1, "1", True, b"1", E.ONE] + + print(T.avro_schema()) + """ + { + "type": "record", + "name": "T", + "fields": [ + { + "name": "f", + "type": [ + "null", + "long", + "string", + "boolean", + "bytes", + { + "type": "enum", + "name": "E", + "symbols": [ + "one" + ] + } + ] + } + ] + } + """ + ``` *(This script is complete, it should run "as is")* diff --git a/docs/index.md b/docs/index.md index 6878bba1..6a7c2093 100644 --- a/docs/index.md +++ b/docs/index.md @@ -35,71 +35,144 @@ To install `avro schemas cli` install [dc-avro](https://marcosschroh.github.io/d ### Generating the avro schema -```python title="Trival Usage" -import enum -import typing -import dataclasses - -from dataclasses_avroschema import AvroModel - - -class FavoriteColor(enum.Enum): - BLUE = "Blue" - YELLOW = "Yellow" - GREEN = "Green" - - -@dataclasses.dataclass -class User(AvroModel): - "An User" - name: str - age: int - pets: typing.List[str] - accounts: typing.Dict[str, int] - favorite_color: FavoriteColor - country: str = "Argentina" - address: str = None - - class Meta: - namespace = "User.v1" - aliases = ["user-v1", "super user"] - - -User.avro_schema() - -'{ - "type": "record", - "name": "User", - "doc": "An User", - "namespace": "User.v1", - "aliases": ["user-v1", "super user"], - "fields": [ - {"name": "name", "type": "string"}, - {"name": "age", "type": "long"}, - {"name": "pets", "type": "array", "items": "string"}, - {"name": "accounts", "type": "map", "values": "long"}, - {"name": "favorite_color", "type": {"type": "enum", "name": "FavoriteColor", "symbols": ["Blue", "Yellow", "Green"]}} - {"name": "country", "type": "string", "default": "Argentina"}, - {"name": "address", "type": ["null", "string"], "default": null} - ] -}' - -User.avro_schema_to_python() - -{ - "type": "record", - "name": "User", - "doc": "An User", - "namespace": "User.v1", - "aliases": ["user-v1", "super user"], - "fields": [ - {"name": "name", "type": "string"}, - {"name": "age", "type": "long"}, - {"name": "pets", "type": {"type": "array", "items": "string", "name": "pet"}}, - {"name": "accounts", "type": {"type": "map", "values": "long", "name": "account"}}, - {"name": "favorite_color", "type": {"type": "enum", "name": "FavoriteColor", "symbols": ["BLUE", "YELLOW", "GREEN"]}}, - {"name": "country", "type": "string", "default": "Argentina"}, - {"name": "address", "type": ["null", "string"], "default": None} - ], -} -``` +=== "python <= 3.10" + + ```python title="Trival Usage" + import enum + import typing + import dataclasses + + from dataclasses_avroschema import AvroModel + + + class FavoriteColor(enum.Enum): + BLUE = "Blue" + YELLOW = "Yellow" + GREEN = "Green" + + + @dataclasses.dataclass + class User(AvroModel): + "An User" + name: str + age: int + pets: typing.List[str] + accounts: typing.Dict[str, int] + favorite_color: FavoriteColor + country: str = "Argentina" + address: str = None + + class Meta: + namespace = "User.v1" + aliases = ["user-v1", "super user"] + + + User.avro_schema() + + '{ + "type": "record", + "name": "User", + "doc": "An User", + "namespace": "User.v1", + "aliases": ["user-v1", "super user"], + "fields": [ + {"name": "name", "type": "string"}, + {"name": "age", "type": "long"}, + {"name": "pets", "type": "array", "items": "string"}, + {"name": "accounts", "type": "map", "values": "long"}, + {"name": "favorite_color", "type": {"type": "enum", "name": "FavoriteColor", "symbols": ["Blue", "Yellow", "Green"]}} + {"name": "country", "type": "string", "default": "Argentina"}, + {"name": "address", "type": ["null", "string"], "default": null} + ] + }' + + User.avro_schema_to_python() + + { + "type": "record", + "name": "User", + "doc": "An User", + "namespace": "User.v1", + "aliases": ["user-v1", "super user"], + "fields": [ + {"name": "name", "type": "string"}, + {"name": "age", "type": "long"}, + {"name": "pets", "type": {"type": "array", "items": "string", "name": "pet"}}, + {"name": "accounts", "type": {"type": "map", "values": "long", "name": "account"}}, + {"name": "favorite_color", "type": {"type": "enum", "name": "FavoriteColor", "symbols": ["BLUE", "YELLOW", "GREEN"]}}, + {"name": "country", "type": "string", "default": "Argentina"}, + {"name": "address", "type": ["null", "string"], "default": None} + ], + } + ``` + +=== "python >= 3.11" + + ```python title="Trival Usage" + import enum + import typing + import dataclasses + + from dataclasses_avroschema import AvroModel + + + class FavoriteColor(str, enum.Enum): + BLUE = "Blue" + YELLOW = "Yellow" + GREEN = "Green" + + + @dataclasses.dataclass + class User(AvroModel): + "An User" + name: str + age: int + pets: typing.List[str] + accounts: typing.Dict[str, int] + favorite_color: FavoriteColor + country: str = "Argentina" + address: str = None + + class Meta: + namespace = "User.v1" + aliases = ["user-v1", "super user"] + + + User.avro_schema() + + '{ + "type": "record", + "name": "User", + "doc": "An User", + "namespace": "User.v1", + "aliases": ["user-v1", "super user"], + "fields": [ + {"name": "name", "type": "string"}, + {"name": "age", "type": "long"}, + {"name": "pets", "type": "array", "items": "string"}, + {"name": "accounts", "type": "map", "values": "long"}, + {"name": "favorite_color", "type": {"type": "enum", "name": "FavoriteColor", "symbols": ["Blue", "Yellow", "Green"]}} + {"name": "country", "type": "string", "default": "Argentina"}, + {"name": "address", "type": ["null", "string"], "default": null} + ] + }' + + User.avro_schema_to_python() + + { + "type": "record", + "name": "User", + "doc": "An User", + "namespace": "User.v1", + "aliases": ["user-v1", "super user"], + "fields": [ + {"name": "name", "type": "string"}, + {"name": "age", "type": "long"}, + {"name": "pets", "type": {"type": "array", "items": "string", "name": "pet"}}, + {"name": "accounts", "type": {"type": "map", "values": "long", "name": "account"}}, + {"name": "favorite_color", "type": {"type": "enum", "name": "FavoriteColor", "symbols": ["BLUE", "YELLOW", "GREEN"]}}, + {"name": "country", "type": "string", "default": "Argentina"}, + {"name": "address", "type": ["null", "string"], "default": None} + ], + } + ``` diff --git a/docs/migration_guide.md b/docs/migration_guide.md index 1d901699..e84152d2 100644 --- a/docs/migration_guide.md +++ b/docs/migration_guide.md @@ -33,39 +33,76 @@ it is explicit. ## Migration from previous versions to 0.27.0 -- `types.Enum` was replaced with `enum.Enum`. You must create your custom enum, example: +=== "python <= 3.10" -```python -import dataclasses -from dataclasses_avroschema import AvroModel, types + - `types.Enum` was replaced with `enum.Enum`. You must create your custom enum, example: + ```python + import dataclasses + from dataclasses_avroschema import AvroModel, types -class UserAdvance(AvroModel): - name: str - age: int - favorite_colors: types.Enum = types.Enum(["BLUE", "YELLOW", "GREEN"], default="BLUE") # --> replace with field!!! -``` -should be replaced by: + class UserAdvance(AvroModel): + name: str + age: int + favorite_colors: types.Enum = types.Enum(["BLUE", "YELLOW", "GREEN"], default="BLUE") # --> replace with field!!! + ``` -```python -import enum -import dataclasses -from dataclasses_avroschema import AvroModel + should be replaced by: + ```python + import enum + import dataclasses + from dataclasses_avroschema import AvroModel -# New enum!! -class FavoriteColor(enum.Enum): - BLUE = "BLUE" - YELLOW = "YELLOW" - GREEN = "GREEN" + class FavoriteColor(enum.Enum): + BLUE = "BLUE" + YELLOW = "YELLOW" + GREEN = "GREEN" -class UserAdvance: - name: str - age: int - favorite_colors: FavoriteColor = FavoriteColor.BLUE # --> field updated!!! -``` + + class UserAdvance: + name: str + age: int + favorite_colors: FavoriteColor = FavoriteColor.BLUE # --> field updated!!! + ``` + +=== "python >= 3.11" + + - `types.Enum` was replaced with `str, enum.Enum`. You must create your custom enum, example: + + ```python + import dataclasses + from dataclasses_avroschema import AvroModel, types + + + class UserAdvance(AvroModel): + name: str + age: int + favorite_colors: types.Enum = types.Enum(["BLUE", "YELLOW", "GREEN"], default="BLUE") # --> replace with field!!! + ``` + + should be replaced by: + + ```python + import enum + import dataclasses + from dataclasses_avroschema import AvroModel + + + # New enum!! + class FavoriteColor(str, enum.Enum): + BLUE = "BLUE" + YELLOW = "YELLOW" + GREEN = "GREEN" + + + class UserAdvance: + name: str + age: int + favorite_colors: FavoriteColor = FavoriteColor.BLUE # --> field updated!!! + ``` ## Migration from previous versions to 0.23.0 diff --git a/docs/model_generator.md b/docs/model_generator.md index 8e6e0662..8eced494 100644 --- a/docs/model_generator.md +++ b/docs/model_generator.md @@ -15,29 +15,57 @@ The rendered result is a string that contains the proper identation, so the resu ## Mapping `avro fields` to `python fields` summary -|Avro Type | Python Type | -|-----------|-------------| -| string | str | -| int | long | -| boolean | bool | -| float | double | -| null | None | -| bytes | bytes | -| array | typing.List | -| map | typing.Dict | -| fixed | types.confixed | -| enum | enum.Enum | -| int | types.Int32 | -| float | types.Float32| -| union | typing.Union| -| record | Python class| -| date | datetime.date| -| time-millis| datetime.time| -| time-micros| types.TimeMicro| -| timestamp-millis| datetime.datetime| -| timestamp-micros| types.DateTimeMicro| -| decimal | types.condecimal| -| uuid | uuid.UUID | +=== "python <= 3.10" + + |Avro Type | Python Type | + |-----------|-------------| + | string | str | + | int | long | + | boolean | bool | + | float | double | + | null | None | + | bytes | bytes | + | array | typing.List | + | map | typing.Dict | + | fixed | types.confixed | + | enum | enum.Enum | + | int | types.Int32 | + | float | types.Float32| + | union | typing.Union| + | record | Python class| + | date | datetime.date| + | time-millis| datetime.time| + | time-micros| types.TimeMicro| + | timestamp-millis| datetime.datetime| + | timestamp-micros| types.DateTimeMicro| + | decimal | types.condecimal| + | uuid | uuid.UUID | + +=== "python >= 3.11" + + |Avro Type | Python Type | + |-----------|-------------| + | string | str | + | int | long | + | boolean | bool | + | float | double | + | null | None | + | bytes | bytes | + | array | typing.List | + | map | typing.Dict | + | fixed | types.confixed | + | enum | str, enum.Enum | + | int | types.Int32 | + | float | types.Float32| + | union | typing.Union| + | record | Python class| + | date | datetime.date| + | time-millis| datetime.time| + | time-micros| types.TimeMicro| + | timestamp-millis| datetime.datetime| + | timestamp-micros| types.DateTimeMicro| + | decimal | types.condecimal| + | uuid | uuid.UUID | ## Usage @@ -489,22 +517,43 @@ with open("models.py", mode="+w") as f: Then the result will be: -```python -# models.py -from dataclasses_avroschema import AvroModel -import dataclasses -import enum +=== "python <= 3.10" + + ```python + # models.py + from dataclasses_avroschema import AvroModel + import dataclasses + import enum -class UnitMultiPlayer(enum.Enum): - Q = "Q" - q = "q" + class UnitMultiPlayer(enum.Enum): + Q = "Q" + q = "q" -@dataclasses.dataclass -class User(AvroModel): - unit_multi_player: UnitMultiPlayer -``` + @dataclasses.dataclass + class User(AvroModel): + unit_multi_player: UnitMultiPlayer + ``` + +=== "python >= 3.11" + + ```python + # models.py + from dataclasses_avroschema import AvroModel + import dataclasses + import enum + + + class UnitMultiPlayer(str, enum.Enum): + Q = "Q" + q = "q" + + + @dataclasses.dataclass + class User(AvroModel): + unit_multi_player: UnitMultiPlayer + ``` As the example shows the second enum member `UnitMultiPlayer.p` is not in uppercase otherwise will collide with the first member `UnitMultiPlayer.P` diff --git a/tests/conftest.py b/tests/conftest.py index 583b9fd7..bf035ebc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,7 +24,7 @@ class FavoriteColor(str, enum.Enum): @pytest.fixture def user_type_enum(): - class UserType(enum.Enum): + class UserType(str, enum.Enum): BASIC = "BASIC" PREMIUM = "PREMIUM" diff --git a/tests/model_generator/test_model_generator.py b/tests/model_generator/test_model_generator.py index 7b281823..ab51235e 100644 --- a/tests/model_generator/test_model_generator.py +++ b/tests/model_generator/test_model_generator.py @@ -2,6 +2,7 @@ from dataclasses_avroschema.fields import field_utils from dataclasses_avroschema.model_generator.lang.python.avro_to_python_utils import ( render_datetime, + templates, ) @@ -27,6 +28,7 @@ class User(AvroModel): is_student: bool = True encoded: bytes = b"Hi" + class Meta: namespace = "test" aliases = ['schema', 'test-schema'] @@ -197,14 +199,14 @@ class User(AvroModel): def test_schema_with_enum_types(schema_with_enum_types: types.JsonDict) -> None: - expected_result = """ + expected_result = f""" from dataclasses_avroschema import AvroModel import dataclasses import enum import typing -class FavoriteColor(enum.Enum): +class FavoriteColor({templates.ENUM_PYTHON_VERSION}): \""" A favorite color \""" @@ -212,18 +214,18 @@ class FavoriteColor(enum.Enum): YELLOW = "Yellow" GREEN = "Green" + {templates.METACLASS_DECORATOR} class Meta: namespace = "some.name.space" aliases = ['Color', 'My favorite color'] - -class Superheros(enum.Enum): +class Superheros({templates.ENUM_PYTHON_VERSION}): BATMAN = "batman" SUPERMAN = "superman" SPIDERMAN = "spiderman" -class Cars(enum.Enum): +class Cars({templates.ENUM_PYTHON_VERSION}): BMW = "bmw" FERRARY = "ferrary" DUNA = "duna" @@ -258,6 +260,7 @@ class DeliveryBatch(AvroModel): teammates: typing.Dict[str, str] = dataclasses.field(metadata={'inner_name': 'my_teammate'}, default_factory=dict) a_fixed: types.confixed(size=16) = dataclasses.field(metadata={'inner_name': 'my_fixed'}, default=b"u00ffffffffffffx") + class Meta: namespace = "app.delivery.email" @@ -270,13 +273,13 @@ class Meta: def test_schema_with_enum_types_case_sensitivity( schema_with_enum_types_case_sensitivity: types.JsonDict, ) -> None: - expected_result = """ + expected_result = f""" from dataclasses_avroschema import AvroModel import dataclasses import enum -class unit_multi_player(enum.Enum): +class unit_multi_player({templates.ENUM_PYTHON_VERSION}): q = "q" Q = "Q" @@ -292,14 +295,14 @@ class User(AvroModel): def test_enum_types_with_no_pascal_case(schema_with_enum_types_no_pascal_case) -> None: - expected_result = ''' + expected_result = f''' from dataclasses_avroschema import AvroModel import dataclasses import enum import typing -class my_favorite_color(enum.Enum): +class my_favorite_color({templates.ENUM_PYTHON_VERSION}): """ A favorite color """ @@ -307,18 +310,18 @@ class my_favorite_color(enum.Enum): YELLOW = "Yellow" GREEN = "Green" + {templates.METACLASS_DECORATOR} class Meta: namespace = "some.name.space" aliases = ['Color', 'My favorite color'] - -class super_heros(enum.Enum): +class super_heros({templates.ENUM_PYTHON_VERSION}): BATMAN = "batman" SUPERMAN = "superman" SPIDERMAN = "spiderman" -class cars(enum.Enum): +class cars({templates.ENUM_PYTHON_VERSION}): BMW = "bmw" FERRARY = "ferrary" DUNA = "duna" @@ -540,6 +543,7 @@ class LogicalTypes(AvroModel): event_uuid: uuid.UUID = "ad0677ab-bd1c-4383-9d45-e46c56bcc5c9" explicit_with_default: types.condecimal(max_digits=3, decimal_places=2) = decimal.Decimal('3.14') + class Meta: field_order = ['uuid_1', 'meeting_date', 'release_date', 'meeting_time', 'release_time', 'release_time_micro', 'meeting_datetime', 'birthday', 'birthday_time', 'birthday_datetime', 'release_datetime', 'release_datetime_micro', 'uuid_2', 'event_uuid', 'explicit_with_default', 'money'] """ @@ -570,11 +574,11 @@ class User(AvroModel): is_student: bool = True encoded: bytes = b"Hi" + class Meta: namespace = "test" aliases = ['schema', 'test-schema'] - @dataclasses.dataclass class Address(AvroModel): \""" @@ -602,6 +606,7 @@ class Message(AvroModel): someotherfield: int = dataclasses.field(metadata={'aliases': ['oldname'], 'doc': 'test'}) fieldwithdefault: str = "some default value" + class Meta: field_order = ['fieldwithdefault', 'someotherfield'] @@ -628,10 +633,10 @@ class Address(AvroModel): street: str street_number: int + class Meta: original_schema = '{"type": "record", "name": "Address", "fields": [{"name": "street", "type": "string"}, {"name": "street_number", "type": "long"}], "doc": "An Address"}' - @dataclasses.dataclass class User(AvroModel): name: str @@ -640,6 +645,7 @@ class User(AvroModel): crazy_union: typing.Union[str, typing.List[Address]] optional_addresses: typing.Optional[typing.List[Address]] = None + class Meta: original_schema = '{"type": "record", "name": "User", "fields": [{"name": "name", "type": "string"}, {"name": "age", "type": "long"}, {"name": "addresses", "type": {"type": "array", "items": {"type": "record", "name": "Address", "fields": [{"name": "street", "type": "string"}, {"name": "street_number", "type": "long"}], "doc": "An Address"}, "name": "address"}}, {"name": "crazy_union", "type": ["string", {"type": "array", "items": "Address", "name": "optional_address"}]}, {"name": "optional_addresses", "type": ["null", {"type": "array", "items": "Address", "name": "optional_address"}], "default": null}]}' """ diff --git a/tests/model_generator/test_model_pydantic_generator.py b/tests/model_generator/test_model_pydantic_generator.py index bbc8e7d1..379060b4 100644 --- a/tests/model_generator/test_model_pydantic_generator.py +++ b/tests/model_generator/test_model_pydantic_generator.py @@ -174,6 +174,7 @@ class Message(AvroBaseModel): someotherfield: int = Field(metadata={'aliases': ['oldname'], 'doc': 'test'}) fieldwithdefault: str = "some default value" + class Meta: field_order = ['fieldwithdefault', 'someotherfield']