From c73ba4af8861e80c1db1eb793e89f19c44a33cdb Mon Sep 17 00:00:00 2001 From: Edwin Yu Date: Tue, 12 Sep 2023 03:34:46 +0000 Subject: [PATCH 1/4] Add MMCloud plugin to Integrations Signed-off-by: Edwin Yu --- docs/index.md | 1 + docs/integrations.md | 2 + examples/mmcloud_plugin/Dockerfile | 27 ++++++ examples/mmcloud_plugin/README.md | 82 +++++++++++++++++++ .../mmcloud_plugin/mmcloud_plugin/__init__.py | 0 .../mmcloud_plugin/mmcloud_plugin/example.py | 42 ++++++++++ examples/mmcloud_plugin/requirements.in | 2 + examples/mmcloud_plugin/requirements.txt | 0 8 files changed, 156 insertions(+) create mode 100644 examples/mmcloud_plugin/Dockerfile create mode 100644 examples/mmcloud_plugin/README.md create mode 100644 examples/mmcloud_plugin/mmcloud_plugin/__init__.py create mode 100644 examples/mmcloud_plugin/mmcloud_plugin/example.py create mode 100644 examples/mmcloud_plugin/requirements.in create mode 100644 examples/mmcloud_plugin/requirements.txt diff --git a/docs/index.md b/docs/index.md index c593455a9..0ebdc325b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -448,6 +448,7 @@ auto_examples/greatexpectations_plugin/index auto_examples/hive_plugin/index auto_examples/k8s_pod_plugin/index auto_examples/mlflow_plugin/index +auto_examples/mmcloud_plugin/index auto_examples/modin_plugin/index auto_examples/kfmpi_plugin/index auto_examples/onnx_plugin/index diff --git a/docs/integrations.md b/docs/integrations.md index e187653de..83bb5e878 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -104,6 +104,8 @@ the Flyte task that use the respective plugin. - Running tasks and workflows on AWS batch service * - {doc}`Hive ` - Run Hive jobs in your workflows. +* - {doc}`MMCloud ` + - Execute tasks using MemVerge Memory Machine Cloud * - {doc}`Snowflake ` - Run Snowflake jobs in your workflows. * - {doc}`Databricks ` diff --git a/examples/mmcloud_plugin/Dockerfile b/examples/mmcloud_plugin/Dockerfile new file mode 100644 index 000000000..ebdcf8f06 --- /dev/null +++ b/examples/mmcloud_plugin/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.11-slim-bookworm +LABEL org.opencontainers.image.source https://github.com/flyteorg/flytesnacks + +WORKDIR /root +ENV VENV /opt/venv +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 +ENV PYTHONPATH /root + +WORKDIR /root + +ENV VENV /opt/venv +# Virtual environment +RUN python3 -m venv ${VENV} +ENV PATH="${VENV}/bin:$PATH" + +# Install Python dependencies +COPY requirements.txt /root +RUN pip install -r /root/requirements.txt + +# Copy the actual code +COPY . /root + +# This tag is supplied by the build script and will be used to determine the version +# when registering tasks, workflows, and launch plans +ARG tag +ENV FLYTE_INTERNAL_IMAGE $tag diff --git a/examples/mmcloud_plugin/README.md b/examples/mmcloud_plugin/README.md new file mode 100644 index 000000000..c20558c93 --- /dev/null +++ b/examples/mmcloud_plugin/README.md @@ -0,0 +1,82 @@ +# Memory Machine Cloud + +```{eval-rst} +.. tags:: AWS, GCP, AliCloud, Integration, Advanced +``` + +[MemVerge](https://memverge.com/) [Memory Machine Cloud](https://www.mmcloud.io/) (MMCloud)—available on AWS, GCP, and AliCloud—empowers users to continuously optimize cloud resources during runtime, safely execute stateful tasks on spot instances, and monitor resource usage in real time. These capabilities make it an excellent fit for long-running batch workloads. + +Flyte can be integrated with MMCloud, allowing you to execute Flyte tasks using MMCloud. + +## Installation + +To install the plugin, run the following command: + +```{eval-rst} +.. prompt:: bash + + pip install flytekitplugins-mmcloud +``` + +To get started with MMCloud, refer to the [MMCloud User Guide](https://docs.memverge.com/mmce/current/userguide/olh/index.html). + +## Configuring the backend to get MMCloud working + +The MMCloud plugin is [enabled in FlytePropeller's configuration](https://docs.flyte.org/en/latest/deployment/plugins/memverge/mmcloud.html). + +## Getting Started + +This plugin allows executing `PythonFunctionTask` using MMCloud without changing any function code. + +```{eval-rst} +.. testcode:: awsbatch-quickstart + from flytekitplugins.mmcloud import MMCloudConfig + + @task(task_config=MMCloudConfig()) + def to_str(i: int) -> str: + return str(i) +``` + +[Resource](https://docs.flyte.org/projects/cookbook/en/latest/auto_examples/productionizing/customizing_resources.html) (cpu and mem) requests and limits, [container](https://docs.flyte.org/projects/cookbook/en/latest/auto_examples/customizing_dependencies/multi_images.html) images, and [environment](https://docs.flyte.org/projects/flytekit/en/latest/generated/flytekit.task.html) variable specifications are supported. + +[ImageSpec](https://docs.flyte.org/projects/cookbook/en/latest/auto_examples/customizing_dependencies/image_spec.html) may be used to define images to run tasks. + +### Credentials + +The following [secrets](https://docs.flyte.org/projects/cookbook/en/latest/auto_examples/productionizing/use_secrets.html) are required to be defined for the agent server: +* `mmc_address`: MMCloud OpCenter address +* `mmc_username`: MMCloud OpCenter username +* `mmc_password`: MMCloud OpCenter password + +### Defaults + +Compute resources: +* If only requests are specified, there are no limits. +* If only limits are specified, the requests are equal to the limits. +* If neither resource requests nor limits are specified, the default requests used for job submission are `cpu="1"` and `mem="1Gi"`, and there are no limits. + +If no container image is specified, the default used for job submission is `DefaultImages.default_image()`. + +### Agent Image + +Install `flytekitplugins-mmcloud` in the agent image. + +A `float` binary (obtainable via the OpCenter) is required. Copy it to the agent image `PATH`. + +Sample `Dockerfile` for building an agent image: +```dockerfile +FROM python:3.11-slim-bookworm + +WORKDIR /root +ENV PYTHONPATH /root + +# flytekit will autoload the agent if package is installed. +RUN pip install flytekitplugins-mmcloud +COPY float /usr/local/bin/float + +CMD pyflyte serve --port 8000 +``` + +```{auto-examples-toc} +example +``` diff --git a/examples/mmcloud_plugin/mmcloud_plugin/__init__.py b/examples/mmcloud_plugin/mmcloud_plugin/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/mmcloud_plugin/mmcloud_plugin/example.py b/examples/mmcloud_plugin/mmcloud_plugin/example.py new file mode 100644 index 000000000..68498b19b --- /dev/null +++ b/examples/mmcloud_plugin/mmcloud_plugin/example.py @@ -0,0 +1,42 @@ +# %% [markdown] +# # Memory Machine Cloud +# +# This example shows how to use the MMCloud plugin to execute tasks on MemVerge Memory Machine Cloud. + +# %% +from flytekit import Resources, task, workflow +from flytekit.configuration import DefaultImages + +from flytekitplugins.mmcloud import MMCloudConfig + +# %% [markdown] +# `MMCloudConfig` configures `MMCloudTask`. Tasks specified with `MMCloudConfig` will be executed using MMCloud. Tasks will be executed with requests `cpu="1"` and `mem="1Gi"` and container image `DefaultImages.default_image()` by default. + +# %% +@task(task_config=MMCloudConfig()) +def to_str(i: int) -> str: + return str(i) + +@task(task_config=MMCloudConfig()) +def to_int(s: str) -> int: + return int(s) + +# %% [markdown] +# [Resource](https://docs.flyte.org/projects/cookbook/en/latest/auto_examples/productionizing/customizing_resources.html) (cpu and mem) requests and limits, [container](https://docs.flyte.org/projects/cookbook/en/latest/auto_examples/customizing_dependencies/multi_images.html) images, and [environment](https://docs.flyte.org/projects/flytekit/en/latest/generated/flytekit.task.html) variable specifications are supported. + +# %% +@task( + task_config=MMCloudConfig(submit_extra="--migratePolicy [enable=true]"), + requests=Resources(cpu="1", mem="1Gi"), + limits=Resources(cpu="2", mem="4Gi"), + container_image=DefaultImages.default_image(), + environment={"KEY": "value"}, +) +def concatenate_str(s1: str, s2: str) -> str: + return s1 + s2 + +@workflow +def concatenate_int_wf(i1: int, i2: int) -> int: + i1_str = to_str(i=i1) + i2_str = to_str(i=i2) + return to_int(s=concatenate_str(s1=i1_str, s2=i2_str)) diff --git a/examples/mmcloud_plugin/requirements.in b/examples/mmcloud_plugin/requirements.in new file mode 100644 index 000000000..153f98cd0 --- /dev/null +++ b/examples/mmcloud_plugin/requirements.in @@ -0,0 +1,2 @@ +flytekitplugins-envd +flytekitplugins-mmcloud \ No newline at end of file diff --git a/examples/mmcloud_plugin/requirements.txt b/examples/mmcloud_plugin/requirements.txt new file mode 100644 index 000000000..e69de29bb From 3cce07804b7a9ba6773f021396621e84f551d181 Mon Sep 17 00:00:00 2001 From: Edwin Yu Date: Wed, 13 Sep 2023 17:50:18 +0000 Subject: [PATCH 2/4] Remove default image from README and example Signed-off-by: Edwin Yu --- examples/mmcloud_plugin/README.md | 2 -- examples/mmcloud_plugin/mmcloud_plugin/example.py | 8 ++++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/examples/mmcloud_plugin/README.md b/examples/mmcloud_plugin/README.md index c20558c93..6a8d3c98d 100644 --- a/examples/mmcloud_plugin/README.md +++ b/examples/mmcloud_plugin/README.md @@ -55,8 +55,6 @@ Compute resources: * If only limits are specified, the requests are equal to the limits. * If neither resource requests nor limits are specified, the default requests used for job submission are `cpu="1"` and `mem="1Gi"`, and there are no limits. -If no container image is specified, the default used for job submission is `DefaultImages.default_image()`. - ### Agent Image Install `flytekitplugins-mmcloud` in the agent image. diff --git a/examples/mmcloud_plugin/mmcloud_plugin/example.py b/examples/mmcloud_plugin/mmcloud_plugin/example.py index 68498b19b..f0f89f21d 100644 --- a/examples/mmcloud_plugin/mmcloud_plugin/example.py +++ b/examples/mmcloud_plugin/mmcloud_plugin/example.py @@ -5,22 +5,22 @@ # %% from flytekit import Resources, task, workflow -from flytekit.configuration import DefaultImages - from flytekitplugins.mmcloud import MMCloudConfig # %% [markdown] -# `MMCloudConfig` configures `MMCloudTask`. Tasks specified with `MMCloudConfig` will be executed using MMCloud. Tasks will be executed with requests `cpu="1"` and `mem="1Gi"` and container image `DefaultImages.default_image()` by default. +# `MMCloudConfig` configures `MMCloudTask`. Tasks specified with `MMCloudConfig` will be executed using MMCloud. Tasks will be executed with requests `cpu="1"` and `mem="1Gi"` by default. # %% @task(task_config=MMCloudConfig()) def to_str(i: int) -> str: return str(i) + @task(task_config=MMCloudConfig()) def to_int(s: str) -> int: return int(s) + # %% [markdown] # [Resource](https://docs.flyte.org/projects/cookbook/en/latest/auto_examples/productionizing/customizing_resources.html) (cpu and mem) requests and limits, [container](https://docs.flyte.org/projects/cookbook/en/latest/auto_examples/customizing_dependencies/multi_images.html) images, and [environment](https://docs.flyte.org/projects/flytekit/en/latest/generated/flytekit.task.html) variable specifications are supported. @@ -29,12 +29,12 @@ def to_int(s: str) -> int: task_config=MMCloudConfig(submit_extra="--migratePolicy [enable=true]"), requests=Resources(cpu="1", mem="1Gi"), limits=Resources(cpu="2", mem="4Gi"), - container_image=DefaultImages.default_image(), environment={"KEY": "value"}, ) def concatenate_str(s1: str, s2: str) -> str: return s1 + s2 + @workflow def concatenate_int_wf(i1: int, i2: int) -> int: i1_str = to_str(i=i1) From 627afe331140bc06ccee8884b8d75d842c178206 Mon Sep 17 00:00:00 2001 From: Edwin Yu Date: Sat, 16 Sep 2023 01:07:28 +0000 Subject: [PATCH 3/4] Fix end of requirements.in Signed-off-by: Edwin Yu --- examples/mmcloud_plugin/requirements.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/mmcloud_plugin/requirements.in b/examples/mmcloud_plugin/requirements.in index 153f98cd0..b490acb14 100644 --- a/examples/mmcloud_plugin/requirements.in +++ b/examples/mmcloud_plugin/requirements.in @@ -1,2 +1,2 @@ flytekitplugins-envd -flytekitplugins-mmcloud \ No newline at end of file +flytekitplugins-mmcloud From c92b593966593db22f51860145a91db8cb1bc949 Mon Sep 17 00:00:00 2001 From: helenzhangyc Date: Thu, 28 Sep 2023 15:46:20 -0700 Subject: [PATCH 4/4] update requirement.txt for mmcloud_plugin Signed-off-by: helenzhangyc --- examples/mmcloud_plugin/requirements.txt | 356 +++++++++++++++++++++++ 1 file changed, 356 insertions(+) diff --git a/examples/mmcloud_plugin/requirements.txt b/examples/mmcloud_plugin/requirements.txt index e69de29bb..1bb46057c 100644 --- a/examples/mmcloud_plugin/requirements.txt +++ b/examples/mmcloud_plugin/requirements.txt @@ -0,0 +1,356 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile requirements.in +# +adlfs==2023.9.0 + # via flytekit +aiobotocore==2.5.4 + # via s3fs +aiohttp==3.8.5 + # via + # adlfs + # aiobotocore + # gcsfs + # s3fs +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp +arrow==1.2.3 + # via cookiecutter +async-timeout==4.0.3 + # via aiohttp +attrs==23.1.0 + # via aiohttp +azure-core==1.29.4 + # via + # adlfs + # azure-identity + # azure-storage-blob +azure-datalake-store==0.0.53 + # via adlfs +azure-identity==1.14.0 + # via adlfs +azure-storage-blob==12.18.2 + # via adlfs +binaryornot==0.4.4 + # via cookiecutter +botocore==1.31.17 + # via aiobotocore +cachetools==5.3.1 + # via google-auth +certifi==2023.7.22 + # via + # kubernetes + # requests +cffi==1.16.0 + # via + # azure-datalake-store + # cryptography +chardet==5.2.0 + # via binaryornot +charset-normalizer==3.2.0 + # via + # aiohttp + # requests +click==8.1.7 + # via + # cookiecutter + # flytekit + # rich-click +cloudpickle==2.2.1 + # via flytekit +cookiecutter==2.3.1 + # via flytekit +croniter==1.4.1 + # via flytekit +cryptography==41.0.4 + # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt + # pyopenssl + # secretstorage +dataclasses-json==0.5.9 + # via flytekit +decorator==5.1.1 + # via gcsfs +deprecated==1.2.14 + # via flytekit +diskcache==5.6.3 + # via flytekit +docker==6.1.3 + # via flytekit +docker-image-py==0.1.12 + # via flytekit +docstring-parser==0.15 + # via flytekit +envd==0.3.40 + # via flytekitplugins-envd +flyteidl==1.5.21 + # via flytekit +flytekit==1.9.1 + # via + # flytekitplugins-envd + # flytekitplugins-mmcloud +flytekitplugins-envd==1.9.1 + # via -r requirements.in +flytekitplugins-mmcloud==1.10.0b0 + # via -r requirements.in +frozenlist==1.4.0 + # via + # aiohttp + # aiosignal +fsspec==2023.9.2 + # via + # adlfs + # flytekit + # gcsfs + # s3fs +gcsfs==2023.9.2 + # via flytekit +gitdb==4.0.10 + # via gitpython +gitpython==3.1.37 + # via flytekit +google-api-core==2.12.0 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.23.2 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # kubernetes +google-auth-oauthlib==1.1.0 + # via gcsfs +google-cloud-core==2.3.3 + # via google-cloud-storage +google-cloud-storage==2.11.0 + # via gcsfs +google-crc32c==1.5.0 + # via google-resumable-media +google-resumable-media==2.6.0 + # via google-cloud-storage +googleapis-common-protos==1.60.0 + # via + # flyteidl + # flytekit + # google-api-core + # grpcio-status +grpcio==1.53.0 + # via + # flytekit + # grpcio-status +grpcio-status==1.53.0 + # via flytekit +idna==3.4 + # via + # requests + # yarl +importlib-metadata==6.8.0 + # via + # flytekit + # keyring +isodate==0.6.1 + # via azure-storage-blob +jaraco-classes==3.3.0 + # via keyring +jeepney==0.8.0 + # via + # keyring + # secretstorage +jinja2==3.1.2 + # via cookiecutter +jmespath==1.0.1 + # via botocore +joblib==1.3.2 + # via flytekit +jsonpickle==3.0.2 + # via flytekit +keyring==24.2.0 + # via flytekit +kubernetes==28.1.0 + # via flytekit +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.3 + # via jinja2 +marshmallow==3.20.1 + # via + # dataclasses-json + # marshmallow-enum + # marshmallow-jsonschema +marshmallow-enum==1.5.1 + # via + # dataclasses-json + # flytekit +marshmallow-jsonschema==0.13.0 + # via flytekit +mdurl==0.1.2 + # via markdown-it-py +more-itertools==10.1.0 + # via jaraco-classes +msal==1.24.0 + # via + # azure-datalake-store + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +multidict==6.0.4 + # via + # aiohttp + # yarl +mypy-extensions==1.0.0 + # via typing-inspect +natsort==8.4.0 + # via flytekit +numpy==1.26.0 + # via + # flytekit + # pandas + # pyarrow +oauthlib==3.2.2 + # via + # kubernetes + # requests-oauthlib +packaging==23.1 + # via + # docker + # marshmallow +pandas==1.5.3 + # via flytekit +portalocker==2.8.2 + # via msal-extensions +protobuf==4.24.3 + # via + # flyteidl + # google-api-core + # googleapis-common-protos + # grpcio-status + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl +pyarrow==10.0.1 + # via flytekit +pyasn1==0.5.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth +pycparser==2.21 + # via cffi +pygments==2.16.1 + # via rich +pyjwt[crypto]==2.8.0 + # via msal +pyopenssl==23.2.0 + # via flytekit +python-dateutil==2.8.2 + # via + # arrow + # botocore + # croniter + # flytekit + # kubernetes + # pandas +python-json-logger==2.0.7 + # via flytekit +python-slugify==8.0.1 + # via cookiecutter +pytimeparse==1.1.8 + # via flytekit +pytz==2023.3.post1 + # via + # flytekit + # pandas +pyyaml==6.0.1 + # via + # cookiecutter + # flytekit + # kubernetes +regex==2023.8.8 + # via docker-image-py +requests==2.31.0 + # via + # azure-core + # azure-datalake-store + # cookiecutter + # docker + # flytekit + # gcsfs + # google-api-core + # google-cloud-storage + # kubernetes + # msal + # requests-oauthlib +requests-oauthlib==1.3.1 + # via + # google-auth-oauthlib + # kubernetes +rich==13.5.3 + # via + # cookiecutter + # flytekit + # rich-click +rich-click==1.6.1 + # via flytekit +rsa==4.9 + # via google-auth +s3fs==2023.9.2 + # via flytekit +secretstorage==3.3.3 + # via keyring +six==1.16.0 + # via + # azure-core + # isodate + # kubernetes + # python-dateutil +smmap==5.0.1 + # via gitdb +sortedcontainers==2.4.0 + # via flytekit +statsd==3.3.0 + # via flytekit +text-unidecode==1.3 + # via python-slugify +typing-extensions==4.8.0 + # via + # aioitertools + # azure-core + # azure-storage-blob + # flytekit + # typing-inspect +typing-inspect==0.9.0 + # via dataclasses-json +urllib3==1.26.16 + # via + # botocore + # docker + # flytekit + # kubernetes + # requests +websocket-client==1.6.3 + # via + # docker + # kubernetes +wheel==0.41.2 + # via flytekit +wrapt==1.15.0 + # via + # aiobotocore + # deprecated + # flytekit +yarl==1.9.2 + # via aiohttp +zipp==3.17.0 + # via importlib-metadata