diff --git a/.github/workflows/merlin.yml b/.github/workflows/merlin.yml index 7348962dd..e716b14f9 100644 --- a/.github/workflows/merlin.yml +++ b/.github/workflows/merlin.yml @@ -42,7 +42,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10"] #TODO: Remove Python 3.7 support env: PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }} steps: @@ -74,7 +74,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10"] #TODO: Remove Python 3.7 support env: PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }} steps: @@ -106,7 +106,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10"] env: PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }} steps: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c3db27c19..a43ceb164 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: 3.8 - name: Install dependencies working-directory: ./python/sdk run: | @@ -46,7 +46,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10"] needs: - publish-python-sdk steps: diff --git a/.readthedocs.yml b/.readthedocs.yml index d684ea5bf..dd6bde34a 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -11,6 +11,6 @@ sphinx: # Optionally set the version of Python and requirements required to build your docs python: - version: 3.7 + version: 3.8 install: - requirements: python/sdk/docs/requirements_docs.txt \ No newline at end of file diff --git a/api/api/versions_api.go b/api/api/versions_api.go index ee8fb8945..08e507930 100644 --- a/api/api/versions_api.go +++ b/api/api/versions_api.go @@ -27,7 +27,7 @@ import ( "github.com/caraml-dev/merlin/utils" ) -const DEFAULT_PYTHON_VERSION = "3.7.*" +const DEFAULT_PYTHON_VERSION = "3.8.*" type VersionsController struct { *AppContext diff --git a/config.yaml b/config.yaml index e60b38379..4bfeeb20f 100644 --- a/config.yaml +++ b/config.yaml @@ -26,7 +26,7 @@ ImageBuilderConfig: CPU: 2 Memory: 1Gi PredictionJobBaseImages: - "3.7.*": + "3.8.*": ImageName: ghcr.io/caraml-dev/merlin-pyspark-base:v0.7.0 DockerfilePath: docker/app.Dockerfile BuildContextURI: "" diff --git a/docs/reference/e2e-test.md b/docs/reference/e2e-test.md index 13d5b76a3..2c1cadeac 100644 --- a/docs/reference/e2e-test.md +++ b/docs/reference/e2e-test.md @@ -11,7 +11,7 @@ Run E2E test as part of github actions workflow. This E2E test will be triggered - Pull merlin repository - Pull mlp repository - Setup go -- Setup python 3.7 +- Setup python 3.8 - Setup cluster - Setup mlp namespace - Deploy mlp diff --git a/docs/user-guide/batch_prediction.md b/docs/user-guide/batch_prediction.md index d5761dfad..3bae730fc 100644 --- a/docs/user-guide/batch_prediction.md +++ b/docs/user-guide/batch_prediction.md @@ -136,7 +136,7 @@ Following error is thrown during batch prediction execution "values") E AttributeError: Can only use .dt accessor with datetimelike values -../../../.local/share/virtualenvs/merlin-pyspark-n4ybPFnE/lib/python3.7/site-packages/pandas/core/indexes/accessors.py:324: AttributeError +../../../.local/share/virtualenvs/merlin-pyspark-n4ybPFnE/lib/python3.8/site-packages/pandas/core/indexes/accessors.py:324: AttributeError Assertion failed ``` diff --git a/examples/batch/env.yaml b/examples/batch/env.yaml index edde1bb72..525de4685 100644 --- a/examples/batch/env.yaml +++ b/examples/batch/env.yaml @@ -2,6 +2,6 @@ dependencies: - pip: - joblib>=0.13.0,<1.2.0 # >=1.2.0 upon upgrade of kserve's version - numpy>=1.19.5 - - scikit-learn==1.0.2 # >=1.1.2 upon python 3.7 deprecation + - scikit-learn==1.0.2 #TODO: >=1.1.2 upon python 3.7 deprecation - xgboost==1.6.2 - mlflow==1.23.0 \ No newline at end of file diff --git a/examples/batch/requirements.txt b/examples/batch/requirements.txt index 8f2429936..867ba1313 100644 --- a/examples/batch/requirements.txt +++ b/examples/batch/requirements.txt @@ -1,6 +1,6 @@ joblib>=0.13.0,<1.2.0 # >=1.2.0 upon upgrade of kserve's version numpy>=1.19.5 -scikit-learn==1.0.2 # >=1.1.2 upon python 3.7 deprecation +scikit-learn==1.0.2 #TODO: >=1.1.2 upon python 3.7 deprecation xgboost==1.6.2 mlflow==1.23.0 merlin-sdk diff --git a/examples/custom-model/http_json/requirements.txt b/examples/custom-model/http_json/requirements.txt index 240522ce6..65c525967 100644 --- a/examples/custom-model/http_json/requirements.txt +++ b/examples/custom-model/http_json/requirements.txt @@ -1,3 +1,3 @@ -scikit-learn==1.0.2 # >=1.1.2 upon python 3.7 deprecation +scikit-learn==1.0.2 #TODO: >=1.1.2 upon python 3.7 deprecation xgboost==1.6.2 merlin-sdk \ No newline at end of file diff --git a/examples/metrics/env.yaml b/examples/metrics/env.yaml index 3ce3ab752..aca49efbb 100644 --- a/examples/metrics/env.yaml +++ b/examples/metrics/env.yaml @@ -1,2 +1,2 @@ dependencies: - - python=3.7 \ No newline at end of file + - python=3.8 \ No newline at end of file diff --git a/examples/model-endpoint/requirements.txt b/examples/model-endpoint/requirements.txt index b4b4486a4..f9f15f6c0 100644 --- a/examples/model-endpoint/requirements.txt +++ b/examples/model-endpoint/requirements.txt @@ -1,4 +1,4 @@ -scikit-learn==1.0.2 # >=1.1.2 upon python 3.7 deprecation +scikit-learn==1.0.2 #TODO: >=1.1.2 upon python 3.7 deprecation merlin-sdk joblib>=0.13.0,<1.2.0 # >=1.2.0 upon upgrade of kserve's version cloudpickle==2.0.0 \ No newline at end of file diff --git a/examples/pyfunc-upi/env.yaml b/examples/pyfunc-upi/env.yaml index 1dace8050..1927d1e20 100644 --- a/examples/pyfunc-upi/env.yaml +++ b/examples/pyfunc-upi/env.yaml @@ -1,5 +1,5 @@ dependencies: - - python=3.7 + - python=3.8 - pip: - numpy - xgboost==1.6.2 \ No newline at end of file diff --git a/examples/pyfunc-upi/requirements.txt b/examples/pyfunc-upi/requirements.txt index 6aaca7c97..5fe961672 100644 --- a/examples/pyfunc-upi/requirements.txt +++ b/examples/pyfunc-upi/requirements.txt @@ -1,4 +1,4 @@ xgboost==1.6.2 -scikit-learn==1.0.2 # >=1.1.2 upon python 3.7 deprecation +scikit-learn==1.0.2 #TODO: >=1.1.2 upon python 3.7 deprecation merlin-sdk>=0.22.2rc4 numpy \ No newline at end of file diff --git a/examples/pyfunc/env.yaml b/examples/pyfunc/env.yaml index 5086d51d8..14f0c61bf 100644 --- a/examples/pyfunc/env.yaml +++ b/examples/pyfunc/env.yaml @@ -1,7 +1,7 @@ dependencies: - - python=3.7 + - python=3.8 - pip: - joblib>=0.13.0,<1.2.0 # >=1.2.0 upon upgrade of kserve's version - numpy - - scikit-learn==1.0.2 # >=1.1.2 upon python 3.7 deprecation + - scikit-learn==1.0.2 #TODO: >=1.1.2 upon python 3.7 deprecation - xgboost==1.6.2 \ No newline at end of file diff --git a/examples/pyfunc/requirements.txt b/examples/pyfunc/requirements.txt index e48da0005..c94dc0bb9 100644 --- a/examples/pyfunc/requirements.txt +++ b/examples/pyfunc/requirements.txt @@ -1,4 +1,4 @@ -scikit-learn==1.0.2 # >=1.1.2 upon python 3.7 deprecation +scikit-learn==1.0.2 #TODO: >=1.1.2 upon python 3.7 deprecation joblib>=0.13.0,<1.2.0 # >=1.2.0 upon upgrade of kserve's version xgboost==1.6.2 merlin-sdk diff --git a/examples/resource-request-gpu/requirements.txt b/examples/resource-request-gpu/requirements.txt index b4b4486a4..f9f15f6c0 100644 --- a/examples/resource-request-gpu/requirements.txt +++ b/examples/resource-request-gpu/requirements.txt @@ -1,4 +1,4 @@ -scikit-learn==1.0.2 # >=1.1.2 upon python 3.7 deprecation +scikit-learn==1.0.2 #TODO: >=1.1.2 upon python 3.7 deprecation merlin-sdk joblib>=0.13.0,<1.2.0 # >=1.2.0 upon upgrade of kserve's version cloudpickle==2.0.0 \ No newline at end of file diff --git a/examples/resource-request/requirements.txt b/examples/resource-request/requirements.txt index b4b4486a4..f9f15f6c0 100644 --- a/examples/resource-request/requirements.txt +++ b/examples/resource-request/requirements.txt @@ -1,4 +1,4 @@ -scikit-learn==1.0.2 # >=1.1.2 upon python 3.7 deprecation +scikit-learn==1.0.2 #TODO: >=1.1.2 upon python 3.7 deprecation merlin-sdk joblib>=0.13.0,<1.2.0 # >=1.2.0 upon upgrade of kserve's version cloudpickle==2.0.0 \ No newline at end of file diff --git a/examples/sklearn/requirements.txt b/examples/sklearn/requirements.txt index b4b4486a4..f9f15f6c0 100644 --- a/examples/sklearn/requirements.txt +++ b/examples/sklearn/requirements.txt @@ -1,4 +1,4 @@ -scikit-learn==1.0.2 # >=1.1.2 upon python 3.7 deprecation +scikit-learn==1.0.2 #TODO: >=1.1.2 upon python 3.7 deprecation merlin-sdk joblib>=0.13.0,<1.2.0 # >=1.2.0 upon upgrade of kserve's version cloudpickle==2.0.0 \ No newline at end of file diff --git a/examples/transformer/custom-transformer/env.yaml b/examples/transformer/custom-transformer/env.yaml index 2da20d4ee..2568f6a93 100644 --- a/examples/transformer/custom-transformer/env.yaml +++ b/examples/transformer/custom-transformer/env.yaml @@ -1,5 +1,5 @@ dependencies: - - python=3.7 + - python=3.8 - pip: - torch==1.3.1 - torchvision==0.4.2 \ No newline at end of file diff --git a/examples/transformer/feast-enricher-transformer/env.yaml b/examples/transformer/feast-enricher-transformer/env.yaml index 3ce3ab752..aca49efbb 100644 --- a/examples/transformer/feast-enricher-transformer/env.yaml +++ b/examples/transformer/feast-enricher-transformer/env.yaml @@ -1,2 +1,2 @@ dependencies: - - python=3.7 \ No newline at end of file + - python=3.8 \ No newline at end of file diff --git a/examples/transformer/standard-transformer/env.yaml b/examples/transformer/standard-transformer/env.yaml index a26454f4a..c9a311f33 100644 --- a/examples/transformer/standard-transformer/env.yaml +++ b/examples/transformer/standard-transformer/env.yaml @@ -1,2 +1,2 @@ dependencies: - - python=3.7 + - python=3.8 diff --git a/examples/transformer/upi/env.yaml b/examples/transformer/upi/env.yaml index a26454f4a..c9a311f33 100644 --- a/examples/transformer/upi/env.yaml +++ b/examples/transformer/upi/env.yaml @@ -1,2 +1,2 @@ dependencies: - - python=3.7 + - python=3.8 diff --git a/examples/xgboost/requirements.txt b/examples/xgboost/requirements.txt index 21b0fb5bf..916acd8da 100644 --- a/examples/xgboost/requirements.txt +++ b/examples/xgboost/requirements.txt @@ -1,4 +1,4 @@ -scikit-learn==1.0.2 # >=1.1.2 upon python 3.7 deprecation-learn +scikit-learn==1.0.2 #TODO: >=1.1.2 upon python 3.7 deprecation xgboost==1.6.2 merlin-sdk cloudpickle==2.0.0 \ No newline at end of file diff --git a/python/batch-predictor/requirements.txt b/python/batch-predictor/requirements.txt index 6cc772441..c7624d8d2 100644 --- a/python/batch-predictor/requirements.txt +++ b/python/batch-predictor/requirements.txt @@ -4,4 +4,5 @@ mlflow>=1.2.0,<=1.23.0 cloudpickle==2.0.0 pyarrow>=0.14.1,<=9.0.0 protobuf>=3.0,<4.0.0 -file:${SDK_PATH}#egg=merlin-sdk +#TODO: Update merlin-sdk dep to: file:${SDK_PATH}#egg=merlin-sdk +merlin-sdk==0.33.0 # Pin to the version that supports Python 3.7. diff --git a/python/batch-predictor/requirements_test.txt b/python/batch-predictor/requirements_test.txt index d15de3a8a..d1bf453d4 100644 --- a/python/batch-predictor/requirements_test.txt +++ b/python/batch-predictor/requirements_test.txt @@ -2,7 +2,7 @@ pytest pytest-cov mypy google-cloud-bigquery -scikit-learn==1.0.2 # >=1.1.2 upon python 3.7 deprecation +scikit-learn==1.0.2 #TODO: >=1.1.2 upon python 3.7 deprecation joblib>=0.13.0,<1.2.0 # >=1.2.0 upon upgrade of kserve's version mypy-protobuf>=1.19 types-PyYAML diff --git a/python/batch-predictor/setup.py b/python/batch-predictor/setup.py index 9f6537f99..f4b10f427 100644 --- a/python/batch-predictor/setup.py +++ b/python/batch-predictor/setup.py @@ -21,11 +21,12 @@ with open('requirements.txt') as f: REQUIRE = f.read().splitlines() -merlin_path = os.path.join(os.getcwd(), "../sdk") -merlin_sdk_package = "merlin-sdk" -for index, item in enumerate(REQUIRE): - if merlin_sdk_package in item: - REQUIRE[index] = f"{merlin_sdk_package} @ file://localhost/{merlin_path}#egg={merlin_sdk_package}" +# TODO: Uncomment below lines after Pyfunc server stops supporting Python 3.7 +# merlin_path = os.path.join(os.getcwd(), "../sdk") +# merlin_sdk_package = "merlin-sdk" +# for index, item in enumerate(REQUIRE): +# if merlin_sdk_package in item: +# REQUIRE[index] = f"{merlin_sdk_package} @ file://localhost/{merlin_path}#egg={merlin_sdk_package}" setup( name='merlin-pyspark-app', diff --git a/python/batch-predictor/test-model/conda.yaml b/python/batch-predictor/test-model/conda.yaml index 333db7806..fcba475e2 100644 --- a/python/batch-predictor/test-model/conda.yaml +++ b/python/batch-predictor/test-model/conda.yaml @@ -1,4 +1,4 @@ dependencies: - pip: - - scikit-learn==1.0.2 # >=1.1.2 upon python 3.7 deprecation + - scikit-learn==1.0.2 #TODO: >=1.1.2 upon python 3.7 deprecation - joblib>=0.13.0,<1.2.0 # >=1.2.0 upon upgrade of kserve's version diff --git a/python/pyfunc-scaffolding/{{cookiecutter.model_slug}}/env/conda.yaml b/python/pyfunc-scaffolding/{{cookiecutter.model_slug}}/env/conda.yaml index a26454f4a..c9a311f33 100644 --- a/python/pyfunc-scaffolding/{{cookiecutter.model_slug}}/env/conda.yaml +++ b/python/pyfunc-scaffolding/{{cookiecutter.model_slug}}/env/conda.yaml @@ -1,2 +1,2 @@ dependencies: - - python=3.7 + - python=3.8 diff --git a/python/pyfunc-server/pyfuncserver/utils/contants.py b/python/pyfunc-server/pyfuncserver/utils/contants.py index 7a196f490..2c233065d 100644 --- a/python/pyfunc-server/pyfuncserver/utils/contants.py +++ b/python/pyfunc-server/pyfuncserver/utils/contants.py @@ -11,14 +11,14 @@ conda env.yaml that you pass to log_pyfunc method, e.g.: dependencies: -- python=3.7 +- python=3.8 - pip: - my-missing-package==1.0 OR if the package is not available in PyPI distribution dependencies: -- python=3.7 +- python=3.8 - my-missing-package=1.0 See: https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#creating-an-environment-from-an-environment-yml-file diff --git a/python/pyfunc-server/requirements.txt b/python/pyfunc-server/requirements.txt index f4c9d4b94..d4e3c3c08 100644 --- a/python/pyfunc-server/requirements.txt +++ b/python/pyfunc-server/requirements.txt @@ -11,4 +11,5 @@ grpcio<1.49.0 grpcio-reflection<1.49.0 grpcio-tools<1.49.0 grpcio-health-checking<1.49.0 -file:${SDK_PATH}#egg=merlin-sdk +#TODO: Update merlin-sdk dep to: file:${SDK_PATH}#egg=merlin-sdk +merlin-sdk==0.33.0 # Pin to the version that supports Python 3.7. diff --git a/python/pyfunc-server/setup.py b/python/pyfunc-server/setup.py index aeff89d02..e25cbb30b 100644 --- a/python/pyfunc-server/setup.py +++ b/python/pyfunc-server/setup.py @@ -30,11 +30,13 @@ # replace merlin relative path in requirements.txt into absolute path # setuptools could not install relative path requirements -merlin_path = os.path.join(os.getcwd(), "../sdk") -merlin_sdk_package = "merlin-sdk" -for index, item in enumerate(REQUIRE): - if merlin_sdk_package in item: - REQUIRE[index] = f"{merlin_sdk_package} @ file://localhost/{merlin_path}#egg={merlin_sdk_package}" + +# TODO: Uncomment below lines after Pyfunc server stops supporting Python 3.7 +# merlin_path = os.path.join(os.getcwd(), "../sdk") +# merlin_sdk_package = "merlin-sdk" +# for index, item in enumerate(REQUIRE): +# if merlin_sdk_package in item: +# REQUIRE[index] = f"{merlin_sdk_package} @ file://localhost/{merlin_path}#egg={merlin_sdk_package}" setup( name='pyfuncserver', diff --git a/python/sdk/Dockerfile b/python/sdk/Dockerfile index ed47d0041..e6dd237b8 100644 --- a/python/sdk/Dockerfile +++ b/python/sdk/Dockerfile @@ -1,4 +1,4 @@ -ARG PYTHON_VERSION=3.7 +ARG PYTHON_VERSION=3.8 FROM python:${PYTHON_VERSION}-slim-buster LABEL org.opencontainers.image.source https://github.com/caraml-dev/merlin diff --git a/python/sdk/setup.py b/python/sdk/setup.py index a6195c92e..4bc97c136 100644 --- a/python/sdk/setup.py +++ b/python/sdk/setup.py @@ -53,7 +53,7 @@ "pytest", "recursive-diff>=1.0.0", "requests", - "scikit-learn==1.0.2", # >=1.1.2 upon python 3.7 deprecation + "scikit-learn==1.0.2", #TODO: >=1.1.2 upon python 3.7 deprecation "types-python-dateutil", "types-PyYAML", "types-six", @@ -76,7 +76,7 @@ setup_requires=["setuptools_scm"], tests_require=TEST_REQUIRES, extras_require={'test': TEST_REQUIRES}, - python_requires='>=3.7,<3.11', + python_requires='>=3.8,<3.11', long_description=open("README.md").read(), long_description_content_type='text/markdown', entry_points=''' diff --git a/python/sdk/test/batch/model/env.yaml b/python/sdk/test/batch/model/env.yaml index bff48f883..659bb6b71 100644 --- a/python/sdk/test/batch/model/env.yaml +++ b/python/sdk/test/batch/model/env.yaml @@ -1,7 +1,7 @@ dependencies: - pip: - mlflow - - scikit-learn==1.0.2 # >=1.1.2 upon python 3.7 deprecation + - scikit-learn==1.0.2 #TODO: >=1.1.2 upon python 3.7 deprecation - joblib>=0.13.0,<1.2.0 # >=1.2.0 upon upgrade of kserve's version - numpy>=1.19.5,<1.23.4 # https://github.com/Azure/MachineLearningNotebooks/issues/1314 - pandas==1.3.5 diff --git a/python/sdk/test/invalid-models/pyfunc-model-invalid/MLmodel b/python/sdk/test/invalid-models/pyfunc-model-invalid/MLmodel index c4c0e7432..9b9b01b98 100644 --- a/python/sdk/test/invalid-models/pyfunc-model-invalid/MLmodel +++ b/python/sdk/test/invalid-models/pyfunc-model-invalid/MLmodel @@ -4,5 +4,5 @@ flavors: env: conda.yaml loader_module: mlflow.pyfunc.model python_model: python_model.pkl - python_version: 3.7.5 + python_version: 3.8.5 utc_time_created: '2020-01-20 09:53:55.355722' diff --git a/python/sdk/test/invalid-models/pyfunc-model-invalid/conda.yaml b/python/sdk/test/invalid-models/pyfunc-model-invalid/conda.yaml index 5e5e00e98..3ab83cd18 100644 --- a/python/sdk/test/invalid-models/pyfunc-model-invalid/conda.yaml +++ b/python/sdk/test/invalid-models/pyfunc-model-invalid/conda.yaml @@ -1,7 +1,7 @@ channels: - defaults dependencies: -- python=3.7.5 +- python=3.8.5 - pip: - mlflow - cloudpickle==2.0.0 diff --git a/python/sdk/test/pyfunc/env.yaml b/python/sdk/test/pyfunc/env.yaml index 586c5eb82..54a476962 100644 --- a/python/sdk/test/pyfunc/env.yaml +++ b/python/sdk/test/pyfunc/env.yaml @@ -3,7 +3,7 @@ dependencies: - pip: - joblib>=0.13.0,<1.2.0 # >=1.2.0 upon upgrade of kserve's version - numpy - - scikit-learn==1.0.2 # >=1.1.2 upon python 3.7 deprecation + - scikit-learn==1.0.2 #TODO: >=1.1.2 upon python 3.7 deprecation - xgboost==1.6.2 - pytest - pytest-xdist==1.34.0 diff --git a/scripts/e2e/values-e2e.yaml b/scripts/e2e/values-e2e.yaml index 95e774964..d59c5f2bb 100644 --- a/scripts/e2e/values-e2e.yaml +++ b/scripts/e2e/values-e2e.yaml @@ -27,14 +27,14 @@ imageBuilder: k8sConfig: {} builderConfig: BaseImages: - 3.7.*: - ImageName: pyfunc-py37:v0.31.3 + 3.8.*: + ImageName: pyfunc-py38:v0.31.3 DockerfilePath: "docker/Dockerfile" BuildContextURI: "git://github.com/gojek/merlin.git#refs/tags/v0.31.3" MainAppPath: /merlin-spark-app/main.py PredictionJobBaseImages: - 3.7.*: - ImageName: pyspark-py37:v0.31.3 + 3.8.*: + ImageName: pyspark-py38:v0.31.3 DockerfilePath: "docker/app.Dockerfile" BuildContextURI: "git://github.com/gojek/merlin.git#refs/tags/v0.31.3" MainAppPath: /merlin-spark-app/main.py