From 1caa00c1fb6649f7ab79976ac9870197471fcc1d Mon Sep 17 00:00:00 2001 From: sumana sree Date: Sun, 20 Oct 2024 22:39:27 +0530 Subject: [PATCH 1/6] Added Readme file in Pydantic integration directory Signed-off-by: sumana sree --- examples/pydantic_integration/README.md | 28 +++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 examples/pydantic_integration/README.md diff --git a/examples/pydantic_integration/README.md b/examples/pydantic_integration/README.md new file mode 100644 index 000000000..c92ac09be --- /dev/null +++ b/examples/pydantic_integration/README.md @@ -0,0 +1,28 @@ +(pydantic_integration)= + +# Pydantic + +```{eval-rst} +.. tags:: Pydantic, Flytekit +``` +[Pydantic](https://pydantic.dev/) is a data validation and settings management library that leverages Python type annotations to enforce type hints at runtime. It provides user-friendly errors when data is invalid, making it easier to ensure data integrity. + +The Flytekit Pydantic plugin adds type support for Pydantic models, enabling seamless integration of data validation and settings management within Flyte tasks and workflows. This documentation demonstrates how to integrate Pydantic with Flytekit using the Flytekit Pydantic plugin. + +## Installation + +To install the Flytekit Pydantic plugin, run the following command: + +``` +pip install flytekitplugins-pydantic +``` +## Example usage + +For a usage example, see {doc}`Pydantic example usage `. + +```{toctree} +:maxdepth: -1 +:hidden: + +example +``` From 5e943e2a5da2398e2eb00bf60c8b673f0950a376 Mon Sep 17 00:00:00 2001 From: sumana sree Date: Sun, 20 Oct 2024 23:09:36 +0530 Subject: [PATCH 2/6] Added requirements.in file in pydantic_plugin directory Signed-off-by: sumana sree --- examples/{pydantic_integration => pydantic_plugin}/README.md | 0 examples/pydantic_plugin/requirements.in | 1 + 2 files changed, 1 insertion(+) rename examples/{pydantic_integration => pydantic_plugin}/README.md (100%) create mode 100644 examples/pydantic_plugin/requirements.in diff --git a/examples/pydantic_integration/README.md b/examples/pydantic_plugin/README.md similarity index 100% rename from examples/pydantic_integration/README.md rename to examples/pydantic_plugin/README.md diff --git a/examples/pydantic_plugin/requirements.in b/examples/pydantic_plugin/requirements.in new file mode 100644 index 000000000..f8ae74bb6 --- /dev/null +++ b/examples/pydantic_plugin/requirements.in @@ -0,0 +1 @@ +flytekitplugins-pydantic From 92d0b3d9f8f9e3bd76e6953651b4b96f49752584 Mon Sep 17 00:00:00 2001 From: sumana sree Date: Sun, 20 Oct 2024 23:19:03 +0530 Subject: [PATCH 3/6] Added Dockerfile in pydantic_plugin folder Signed-off-by: sumana sree --- examples/pydantic_plugin/Dockerfile | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 examples/pydantic_plugin/Dockerfile diff --git a/examples/pydantic_plugin/Dockerfile b/examples/pydantic_plugin/Dockerfile new file mode 100644 index 000000000..4881fd978 --- /dev/null +++ b/examples/pydantic_plugin/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.11-slim-buster +LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks + +WORKDIR /root +ENV VENV /opt/venv +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 +ENV PYTHONPATH /root + +# Install Python dependencies +COPY requirements.in /root +RUN pip install -r /root/requirements.in + +# Copy the actual code +COPY . /root/ + +# This tag is supplied by the build script and will be used to determine the version +# when registering tasks, workflows, and launch plans +ARG tag +ENV FLYTE_INTERNAL_IMAGE $tag From 8ff18ef68320a630330d151d14269f1e344cbba5 Mon Sep 17 00:00:00 2001 From: sumana sree Date: Sun, 20 Oct 2024 23:57:03 +0530 Subject: [PATCH 4/6] added example page to the pydantic_plugin Signed-off-by: sumana sree --- examples/pydantic_plugin/README.md | 2 +- .../pydantic_integration_example.py | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 examples/pydantic_plugin/pydantic_plugin/pydantic_integration_example.py diff --git a/examples/pydantic_plugin/README.md b/examples/pydantic_plugin/README.md index c92ac09be..45bdd8119 100644 --- a/examples/pydantic_plugin/README.md +++ b/examples/pydantic_plugin/README.md @@ -18,7 +18,7 @@ pip install flytekitplugins-pydantic ``` ## Example usage -For a usage example, see {doc}`Pydantic example usage `. +For a usage example, see {doc}`Pydantic example usage `. ```{toctree} :maxdepth: -1 diff --git a/examples/pydantic_plugin/pydantic_plugin/pydantic_integration_example.py b/examples/pydantic_plugin/pydantic_plugin/pydantic_integration_example.py new file mode 100644 index 000000000..0c52f58c8 --- /dev/null +++ b/examples/pydantic_plugin/pydantic_plugin/pydantic_integration_example.py @@ -0,0 +1,45 @@ +# %% [markdown] +# (pydantic_integration_example)= +# +# # Pydantic Integration Example +# +# Pydantic is a data validation and settings management library for Python, enabling the creation of data models with type annotations. +# +# Flyte leverages Pydantic for robust input validation and serialization, ensuring that task inputs are correctly structured. + +# %% +from pydantic.v1 import BaseModel +from flytekit import task, workflow +from flytekit.types.file import FlyteFile +from typing import List + +# %% [markdown] +# Let's first define a Pydantic model for training configuration. +# %% +class TrainConfig(BaseModel): + lr: float = 1e-3 # Learning rate + batch_size: int = 32 # Batch size for training + files: List[FlyteFile] # List of file inputs for training + +# %% [markdown] +# Next, we use the Pydantic model in a Flyte task to train a model. +# %% +@task +def train(cfg: TrainConfig): + print(f"Training with learning rate: {cfg.lr} and batch size: {cfg.batch_size}") + for file in cfg.files: + print(f"Processing file: {file}") + +# %% [markdown] +# Now we define a Flyte workflow that utilizes the training task. +# %% +@workflow +def training_workflow(lr: float = 1e-3, batch_size: int = 32, files: List[FlyteFile] = []): + cfg = TrainConfig(lr=lr, batch_size=batch_size, files=files) + train(cfg=cfg) + +# %% [markdown] +# Finally, we execute the workflow with sample parameters. +# %% +if __name__ == "__main__": + training_workflow(lr=1e-3, batch_size=32, files=[FlyteFile(path="path/to/your/file")]) From 23e3696b642811cd7bdd15bc32e6c5af1fabd8f7 Mon Sep 17 00:00:00 2001 From: sumana sree Date: Mon, 21 Oct 2024 10:07:33 +0530 Subject: [PATCH 5/6] fixed lint errors Signed-off-by: sumana sree --- examples/pydantic_plugin/README.md | 4 ++-- .../pydantic_plugin/pydantic_integration_example.py | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/pydantic_plugin/README.md b/examples/pydantic_plugin/README.md index 45bdd8119..f53ab0007 100644 --- a/examples/pydantic_plugin/README.md +++ b/examples/pydantic_plugin/README.md @@ -5,11 +5,11 @@ ```{eval-rst} .. tags:: Pydantic, Flytekit ``` -[Pydantic](https://pydantic.dev/) is a data validation and settings management library that leverages Python type annotations to enforce type hints at runtime. It provides user-friendly errors when data is invalid, making it easier to ensure data integrity. +[Pydantic](https://pydantic.dev/) is a data validation and settings management library that leverages Python type annotations to enforce type hints at runtime. It provides user-friendly errors when data is invalid, making it easier to ensure data integrity. The Flytekit Pydantic plugin adds type support for Pydantic models, enabling seamless integration of data validation and settings management within Flyte tasks and workflows. This documentation demonstrates how to integrate Pydantic with Flytekit using the Flytekit Pydantic plugin. -## Installation +## Installation To install the Flytekit Pydantic plugin, run the following command: diff --git a/examples/pydantic_plugin/pydantic_plugin/pydantic_integration_example.py b/examples/pydantic_plugin/pydantic_plugin/pydantic_integration_example.py index 0c52f58c8..23a253829 100644 --- a/examples/pydantic_plugin/pydantic_plugin/pydantic_integration_example.py +++ b/examples/pydantic_plugin/pydantic_plugin/pydantic_integration_example.py @@ -8,10 +8,12 @@ # Flyte leverages Pydantic for robust input validation and serialization, ensuring that task inputs are correctly structured. # %% -from pydantic.v1 import BaseModel +from typing import List + from flytekit import task, workflow from flytekit.types.file import FlyteFile -from typing import List +from pydantic.v1 import BaseModel + # %% [markdown] # Let's first define a Pydantic model for training configuration. @@ -21,6 +23,7 @@ class TrainConfig(BaseModel): batch_size: int = 32 # Batch size for training files: List[FlyteFile] # List of file inputs for training + # %% [markdown] # Next, we use the Pydantic model in a Flyte task to train a model. # %% @@ -30,6 +33,7 @@ def train(cfg: TrainConfig): for file in cfg.files: print(f"Processing file: {file}") + # %% [markdown] # Now we define a Flyte workflow that utilizes the training task. # %% @@ -38,6 +42,7 @@ def training_workflow(lr: float = 1e-3, batch_size: int = 32, files: List[FlyteF cfg = TrainConfig(lr=lr, batch_size=batch_size, files=files) train(cfg=cfg) + # %% [markdown] # Finally, we execute the workflow with sample parameters. # %% From 0614fc386734e9ffcd963c88e6d17a7b5eb368b5 Mon Sep 17 00:00:00 2001 From: Sumana Sree Angajala <110307215+sumana-2705@users.noreply.github.com> Date: Thu, 24 Oct 2024 11:05:32 +0530 Subject: [PATCH 6/6] Apply suggestions from code review Co-authored-by: Samhita Alla Signed-off-by: Sumana Sree Angajala <110307215+sumana-2705@users.noreply.github.com> --- examples/pydantic_plugin/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pydantic_plugin/README.md b/examples/pydantic_plugin/README.md index f53ab0007..13b3f3b5d 100644 --- a/examples/pydantic_plugin/README.md +++ b/examples/pydantic_plugin/README.md @@ -7,7 +7,7 @@ ``` [Pydantic](https://pydantic.dev/) is a data validation and settings management library that leverages Python type annotations to enforce type hints at runtime. It provides user-friendly errors when data is invalid, making it easier to ensure data integrity. -The Flytekit Pydantic plugin adds type support for Pydantic models, enabling seamless integration of data validation and settings management within Flyte tasks and workflows. This documentation demonstrates how to integrate Pydantic with Flytekit using the Flytekit Pydantic plugin. +The Flytekit Pydantic plugin adds type support for Pydantic models, enabling integration of data validation within Flyte tasks and workflows. ## Installation