diff --git a/examples/pyfunc/Pyfunc.ipynb b/examples/pyfunc/Pyfunc.ipynb index 7fad7fb0b..ae0efbed0 100644 --- a/examples/pyfunc/Pyfunc.ipynb +++ b/examples/pyfunc/Pyfunc.ipynb @@ -35,7 +35,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -142,9 +142,24 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[12:10:08] WARNING: /Users/runner/work/xgboost/xgboost/python-package/build/temp.macosx-10.9-x86_64-cpython-37/xgboost/src/learner.cc:627: \n", + "Parameters: { \"silent\" } might not be used.\n", + "\n", + " This could be a false alarm, with some parameters getting used by language bindings but\n", + " then being mistakenly passed down to XGBoost core, or some parameter actually being used\n", + " but getting flagged wrongly here. Please open an issue if you find any such cases.\n", + "\n", + "\n" + ] + } + ], "source": [ "model_1_dir = \"xgboost-model\"\n", "BST_FILE = \"model_1.bst\"\n", @@ -174,9 +189,20 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['sklearn-model/model_2.joblib']" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "model_2_dir = \"sklearn-model\"\n", "MODEL_FILE = \"model_2.joblib\"\n", @@ -209,7 +235,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -240,15 +266,110 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'predictions': [[0.22366186074437142, 0.3047382124338789, 0.4715999044700078],\n", + " [0.43930651004457333, 0.2117506019399637, 0.3489429066419144]]}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "m = EnsembleModel()\n", "m.initialize({\"xgb_model\": model_1_path, \"sklearn_model\": model_2_path})\n", "m.infer({\"instances\": [[1,2,3,4], [2,1,2,4]] })" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.4 Run PyFunc Model Server Locally (Optional)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For faster development iteration, you can also simulate running the PyFunc model server in your local machine, by calling `merlin.run_pyfunc_model` function. `run_pyfunc_model` is a blocking function that will start a PyFunc model server in port 8080 by default.\n", + "\n", + "As preqrequisites, you need to have Docker installed in your machine." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024/01/15 12:10:13 INFO mlflow.tracking.fluent: Experiment with name 'ensemblemodel' does not exist. Creating a new experiment.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logging model to local MLflow\n", + "Building Docker image ensemblemodel-dev\n", + "Running PyFunc local server\n", + "b'/opt/conda/envs/merlin-model/lib/python3.8/site-packages/sklearn/base.py:348: InconsistentVersionWarning: Trying to unpickle estimator SVC from version 1.0.2 when using version 1.3.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\\n'\n", + "b'https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations\\n'\n", + "b' warnings.warn(\\n'\n", + "b'INFO:root:Registering model: name: ensemblemodel, version: dev, fullname: ensemblemodel-dev\\n'\n", + "b'INFO:root:Listening on port 8080\\n'\n", + "b'INFO:root:Will fork 1 workers\\n'\n" + ] + } + ], + "source": [ + "merlin.run_pyfunc_model(\n", + " model_instance=EnsembleModel(), \n", + " conda_env=\"env.yaml\", \n", + " artifacts={\"xgb_model\": model_1_path, \"sklearn_model\": model_2_path},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Once the PyFunc model server is running, you will see a log like this:\n", + "\n", + "```\n", + "Running PyFunc local server\n", + "b'INFO:root:Registering model: name: ensemblemodel, version: dev, fullname: ensemblemodel-dev\\n'\n", + "b'INFO:root:Listening on port 8080\\n'\n", + "b'INFO:root:Will fork 1 workers\\n'\n", + "```\n", + "\n", + "Now, you can send the request using curl from your terminal:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "curl -X POST \"http://localhost:8080/v1/models/ensemblemodel-dev:predict\" -d '{\n", + " \"instances\": [\n", + " [2.8, 1.0, 6.8, 0.4],\n", + " [3.1, 1.4, 4.5, 1.6]\n", + " ]\n", + "}'" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -404,9 +525,47 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logging model to local MLflow\n", + "Building Docker image ensemblemodel-dev\n", + "Running PyFunc local server\n", + "b'2024/01/09 06:58:58 WARNING mlflow.pyfunc: The version of Python that the model was saved in, `Python 3.10.13`, differs from the version of Python that is currently running, `Python 3.8.18`, and may be incompatible\\n'\n", + "b'INFO:root:Registering model: name: ensemblemodel, version: dev, fullname: ensemblemodel-dev\\n'\n", + "b'INFO:root:Listening on port 8080\\n'\n", + "b'INFO:root:Will fork 1 workers\\n'\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[5], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mmerlin\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_pyfunc_model\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2\u001b[0m \u001b[43m \u001b[49m\u001b[43mmodel_instance\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mEnsembleModel\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m \u001b[49m\u001b[43mconda_env\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43menv.yaml\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[43m \u001b[49m\u001b[43martifacts\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mxgb_model\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel_1_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43msklearn_model\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel_2_path\u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/merlin/pyfunc.py:448\u001b[0m, in \u001b[0;36mrun_pyfunc_model\u001b[0;34m(model_instance, conda_env, code_dir, artifacts, pyfunc_base_image, port, env_vars, protocol, debug)\u001b[0m\n\u001b[1;32m 444\u001b[0m shutil\u001b[38;5;241m.\u001b[39mcopy(conda_env, dependencies_path)\n\u001b[1;32m 446\u001b[0m artifact_path \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmodel_info\u001b[38;5;241m.\u001b[39mrun_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/artifacts/model\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 448\u001b[0m \u001b[43mrun_pyfunc_local_server\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 449\u001b[0m \u001b[43m \u001b[49m\u001b[43mcontext_path\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontext_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 450\u001b[0m \u001b[43m \u001b[49m\u001b[43mdependencies_path\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdependencies_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 451\u001b[0m \u001b[43m \u001b[49m\u001b[43martifact_path\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43martifact_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 452\u001b[0m \u001b[43m \u001b[49m\u001b[43mmodel_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmodel_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 453\u001b[0m \u001b[43m \u001b[49m\u001b[43mmodel_version\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdev\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 454\u001b[0m \u001b[43m \u001b[49m\u001b[43mpyfunc_base_image\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpyfunc_base_image\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 455\u001b[0m \u001b[43m \u001b[49m\u001b[43mport\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mport\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 456\u001b[0m \u001b[43m \u001b[49m\u001b[43menv_vars\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43menv_vars\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 457\u001b[0m \u001b[43m \u001b[49m\u001b[43mprotocol\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprotocol\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 458\u001b[0m \u001b[43m \u001b[49m\u001b[43mdebug\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdebug\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 459\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/merlin/pyfunc.py:506\u001b[0m, in \u001b[0;36mrun_pyfunc_local_server\u001b[0;34m(context_path, dependencies_path, artifact_path, model_name, model_version, pyfunc_base_image, port, env_vars, protocol, debug)\u001b[0m\n\u001b[1;32m 495\u001b[0m _build_image(\n\u001b[1;32m 496\u001b[0m image_tag\u001b[38;5;241m=\u001b[39mimage_tag,\n\u001b[1;32m 497\u001b[0m context_path\u001b[38;5;241m=\u001b[39mcontext_path,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 502\u001b[0m debug\u001b[38;5;241m=\u001b[39mdebug,\n\u001b[1;32m 503\u001b[0m )\n\u001b[1;32m 505\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRunning PyFunc local server\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 506\u001b[0m \u001b[43m_run_container\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 507\u001b[0m \u001b[43m \u001b[49m\u001b[43mimage_tag\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mimage_tag\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 508\u001b[0m \u001b[43m \u001b[49m\u001b[43mmodel_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmodel_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 509\u001b[0m \u001b[43m \u001b[49m\u001b[43mmodel_version\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmodel_version\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 510\u001b[0m \u001b[43m \u001b[49m\u001b[43mmodel_full_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mmodel_name\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m-\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mmodel_version\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 511\u001b[0m \u001b[43m \u001b[49m\u001b[43mport\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mport\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 512\u001b[0m \u001b[43m \u001b[49m\u001b[43menv_vars\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43menv_vars\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 513\u001b[0m \u001b[43m \u001b[49m\u001b[43mprotocol\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprotocol\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 514\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/merlin/pyfunc.py:597\u001b[0m, in \u001b[0;36m_run_container\u001b[0;34m(image_tag, model_name, model_version, model_full_name, port, env_vars, protocol)\u001b[0m\n\u001b[1;32m 586\u001b[0m container \u001b[38;5;241m=\u001b[39m docker_client\u001b[38;5;241m.\u001b[39mcontainers\u001b[38;5;241m.\u001b[39mrun(\n\u001b[1;32m 587\u001b[0m image\u001b[38;5;241m=\u001b[39mimage_tag,\n\u001b[1;32m 588\u001b[0m name\u001b[38;5;241m=\u001b[39mmodel_name,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 593\u001b[0m remove\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 594\u001b[0m )\n\u001b[1;32m 596\u001b[0m \u001b[38;5;66;03m# continously print docker log until the process is interrupted\u001b[39;00m\n\u001b[0;32m--> 597\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m log \u001b[38;5;129;01min\u001b[39;00m container\u001b[38;5;241m.\u001b[39mlogs(stream\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m):\n\u001b[1;32m 598\u001b[0m \u001b[38;5;28mprint\u001b[39m(log)\n\u001b[1;32m 599\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n", + "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/docker/types/daemon.py:29\u001b[0m, in \u001b[0;36mCancellableStream.__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__next__\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 28\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 29\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mnext\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_stream\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 30\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m urllib3\u001b[38;5;241m.\u001b[39mexceptions\u001b[38;5;241m.\u001b[39mProtocolError:\n\u001b[1;32m 31\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m\n", + "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/docker/api/client.py:386\u001b[0m, in \u001b[0;36mAPIClient._multiplexed_response_stream_helper\u001b[0;34m(self, response)\u001b[0m\n\u001b[1;32m 383\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_disable_socket_timeout(socket)\n\u001b[1;32m 385\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 386\u001b[0m header \u001b[38;5;241m=\u001b[39m \u001b[43mresponse\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraw\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mSTREAM_HEADER_SIZE_BYTES\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 387\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m header:\n\u001b[1;32m 388\u001b[0m \u001b[38;5;28;01mbreak\u001b[39;00m\n", + "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/urllib3/response.py:879\u001b[0m, in \u001b[0;36mHTTPResponse.read\u001b[0;34m(self, amt, decode_content, cache_content)\u001b[0m\n\u001b[1;32m 876\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decoded_buffer) \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m amt:\n\u001b[1;32m 877\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decoded_buffer\u001b[38;5;241m.\u001b[39mget(amt)\n\u001b[0;32m--> 879\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_raw_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 881\u001b[0m flush_decoder \u001b[38;5;241m=\u001b[39m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m (amt \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m data)\n\u001b[1;32m 883\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m data \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decoded_buffer) \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n", + "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/urllib3/response.py:814\u001b[0m, in \u001b[0;36mHTTPResponse._raw_read\u001b[0;34m(self, amt)\u001b[0m\n\u001b[1;32m 811\u001b[0m fp_closed \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fp, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mclosed\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m 813\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_error_catcher():\n\u001b[0;32m--> 814\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fp_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m fp_closed \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 815\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m amt \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m data:\n\u001b[1;32m 816\u001b[0m \u001b[38;5;66;03m# Platform-specific: Buggy versions of Python.\u001b[39;00m\n\u001b[1;32m 817\u001b[0m \u001b[38;5;66;03m# Close the connection when no data is returned\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 822\u001b[0m \u001b[38;5;66;03m# not properly close the connection in all cases. There is\u001b[39;00m\n\u001b[1;32m 823\u001b[0m \u001b[38;5;66;03m# no harm in redundantly calling close.\u001b[39;00m\n\u001b[1;32m 824\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fp\u001b[38;5;241m.\u001b[39mclose()\n", + "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/urllib3/response.py:799\u001b[0m, in \u001b[0;36mHTTPResponse._fp_read\u001b[0;34m(self, amt)\u001b[0m\n\u001b[1;32m 796\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m buffer\u001b[38;5;241m.\u001b[39mgetvalue()\n\u001b[1;32m 797\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 798\u001b[0m \u001b[38;5;66;03m# StringIO doesn't like amt=None\u001b[39;00m\n\u001b[0;32m--> 799\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mif\u001b[39;00m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fp\u001b[38;5;241m.\u001b[39mread()\n", + "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/http/client.py:460\u001b[0m, in \u001b[0;36mHTTPResponse.read\u001b[0;34m(self, amt)\u001b[0m\n\u001b[1;32m 457\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 459\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mchunked:\n\u001b[0;32m--> 460\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_chunked\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 462\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 463\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlength \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m amt \u001b[38;5;241m>\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlength:\n\u001b[1;32m 464\u001b[0m \u001b[38;5;66;03m# clip the read to the \"end of response\"\u001b[39;00m\n", + "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/http/client.py:583\u001b[0m, in \u001b[0;36mHTTPResponse._read_chunked\u001b[0;34m(self, amt)\u001b[0m\n\u001b[1;32m 581\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 582\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 583\u001b[0m chunk_left \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_chunk_left\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 584\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m chunk_left \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 585\u001b[0m \u001b[38;5;28;01mbreak\u001b[39;00m\n", + "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/http/client.py:566\u001b[0m, in \u001b[0;36mHTTPResponse._get_chunk_left\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 564\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_safe_read(\u001b[38;5;241m2\u001b[39m) \u001b[38;5;66;03m# toss the CRLF at the end of the chunk\u001b[39;00m\n\u001b[1;32m 565\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 566\u001b[0m chunk_left \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_next_chunk_size\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 567\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m:\n\u001b[1;32m 568\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m IncompleteRead(\u001b[38;5;124mb\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m)\n", + "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/http/client.py:526\u001b[0m, in \u001b[0;36mHTTPResponse._read_next_chunk_size\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 524\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_read_next_chunk_size\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 525\u001b[0m \u001b[38;5;66;03m# Read the next chunk size from the file\u001b[39;00m\n\u001b[0;32m--> 526\u001b[0m line \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreadline\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_MAXLINE\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 527\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(line) \u001b[38;5;241m>\u001b[39m _MAXLINE:\n\u001b[1;32m 528\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m LineTooLong(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mchunk size\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/socket.py:705\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m 703\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m 704\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 705\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 706\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m timeout:\n\u001b[1;32m 707\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_timeout_occurred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], "source": [ "merlin.run_pyfunc_model(\n", " model_instance=EnsembleModel(), \n", @@ -448,7 +607,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.8.18" } }, "nbformat": 4, diff --git a/examples/pyfunc/sklearn-model/model_2.joblib b/examples/pyfunc/sklearn-model/model_2.joblib index 22280815b..5754bd6a0 100644 Binary files a/examples/pyfunc/sklearn-model/model_2.joblib and b/examples/pyfunc/sklearn-model/model_2.joblib differ diff --git a/examples/pyfunc/xgboost-model/model_1.bst b/examples/pyfunc/xgboost-model/model_1.bst index 302803341..2a59ef64d 100644 Binary files a/examples/pyfunc/xgboost-model/model_1.bst and b/examples/pyfunc/xgboost-model/model_1.bst differ diff --git a/python/pyfunc-server/README.md b/python/pyfunc-server/README.md index 713f004de..6dc4cacf3 100644 --- a/python/pyfunc-server/README.md +++ b/python/pyfunc-server/README.md @@ -5,15 +5,17 @@ It leverages mlflow.pyfunc model for model loading. ## Usage -### HTTP Server +### HTTP Server Run following command to load sample `echo-model` model and start HTTP server: + ```bash PROMETHEUS_MULTIPROC_DIR=prometheus \ python -m pyfuncserver --model_dir echo-model/model ``` This will start http server at port 8080 which you can test using curl command + ```bash curl localhost:8080/v1/models/model-1:predict -H "Content-Type: application/json" -d '{}' ``` @@ -21,19 +23,19 @@ curl localhost:8080/v1/models/model-1:predict -H "Content-Type: application/json ### UPI V1 Server Run following command to load sample `echo-model` model and start UPI v1 server: + ```bash PROMETHEUS_MULTIPROC_DIR=prometheus \ CARAML_PROTOCOL=UPI_V1 \ WORKERS=2 python -m pyfuncserver --model_dir echo-model/model ``` - Since UPI v1 interface is gRPC then you can use grpcurl to send request + ```bash grpcurl -plaintext -d '{}' localhost:9000 caraml.upi.v1.UniversalPredictionService/PredictValues ``` - ## Development Requirements: @@ -54,48 +56,29 @@ make test ``` To run benchmark -```bash -make benchmark -``` - -## Building Docker Image - -To create docker image locally you'll need to first download model artifact. ```bash -gsutil cp -r gs://bucket-name/mlflow/11/68eb8538374c4053b3ecad99a44170bd/artifacts/model . -``` - -Build the docker image - -```bash -docker build -t mymodel:latest -f docker/local.Dockerfile . -``` - -And run the model service - -```bash -docker run -e MODEL_NAME=model -p 8080:8080 mymodel:latest +make benchmark ``` ## Configuration Pyfunc server can be configured via following environment variables -| Environment Variable | Description | -| ------------------------- |--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| CARAML_PROTOCOL | Protocol to be used, the valid values are `HTTP_JSON` and `UPI_V1` | -| CARAML_HTTP_PORT | Pyfunc server will start http server listening to this port when `CARAML_PROTOCOL` = `HTTP_JSON` | -| CARAML_GRPC_PORT | Pyfunc server will start grpc server listening to this port when `CARAML_PROTOCOL` = `UPI_V1` | -| CARAML_MODEL_NAME | Model name | -| CARAML_MODEL_VERSION | Model version | -| CARAML_MODEL_FULL_NAME | Model full name in the format of `${CARAML_MODEL_NAME}-${CARAML_MODEL_FULL_NAME}` | -| WORKERS | Number of Python processes that will be created to allow multi processing (default = 1) | -| LOG_LEVEL | Log level, valid values are `INFO`, `ERROR`, `DEBUG`, `WARN`, `CRITICAL` (default='INFO') | -| GRPC_OPTIONS | GRPC options to configure UPI server as json string. The possible options can be found in [grpc_types.h](https://github.com/grpc/grpc/blob/v1.46.x/include/grpc/impl/codegen/grpc_types.h). Example: '{"grpc.max_concurrent_streams":100}' | -| GRPC_CONCURRENCY | Size of grpc handler threadpool per worker (default = 10) | -| PUSHGATEWAY_ENABLED | Enable pushing metrics to prometheus push gateway, only available when `CARAML_PROTOCOL` is set to `UPI_V1` (default = false) | -| PUSHGATEWAY_URL | Url of the prometheus push gateway (default = localhost:9091) | +| Environment Variable | Description | +| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| CARAML_PROTOCOL | Protocol to be used, the valid values are `HTTP_JSON` and `UPI_V1` | +| CARAML_HTTP_PORT | Pyfunc server will start http server listening to this port when `CARAML_PROTOCOL` = `HTTP_JSON` | +| CARAML_GRPC_PORT | Pyfunc server will start grpc server listening to this port when `CARAML_PROTOCOL` = `UPI_V1` | +| CARAML_MODEL_NAME | Model name | +| CARAML_MODEL_VERSION | Model version | +| CARAML_MODEL_FULL_NAME | Model full name in the format of `${CARAML_MODEL_NAME}-${CARAML_MODEL_FULL_NAME}` | +| WORKERS | Number of Python processes that will be created to allow multi processing (default = 1) | +| LOG_LEVEL | Log level, valid values are `INFO`, `ERROR`, `DEBUG`, `WARN`, `CRITICAL` (default='INFO') | +| GRPC_OPTIONS | GRPC options to configure UPI server as json string. The possible options can be found in [grpc_types.h](https://github.com/grpc/grpc/blob/v1.46.x/include/grpc/impl/codegen/grpc_types.h). Example: '{"grpc.max_concurrent_streams":100}' | +| GRPC_CONCURRENCY | Size of grpc handler threadpool per worker (default = 10) | +| PUSHGATEWAY_ENABLED | Enable pushing metrics to prometheus push gateway, only available when `CARAML_PROTOCOL` is set to `UPI_V1` (default = false) | +| PUSHGATEWAY_URL | Url of the prometheus push gateway (default = localhost:9091) | | PUSHGATEWAY_PUSH_INTERVAL_SEC | Interval in seconds for pushing metrics to prometheus push gateway (default = 30) | ## Directory Structure @@ -104,10 +87,8 @@ Pyfunc server can be configured via following environment variables ├── benchmark <- Benchmarking artifacts ├── docker <- Dockerfiles and environment files ├── Dockerfile <- Dockerfile that will be used by kaniko to build user image in the cluster - ├── base.Dockerfile <- Base docker image that will be used by `Dockerfile` and `local.Dockerfile` - ├── local.Dockerfile <- Dockerfile that can be used to perform local testing - ├── envXY.yaml <- Conda environment for python version X.Y that will be created within `base.Dockerfile` -├── echo-model <- Simple model for testing + ├── base.Dockerfile <- Base docker image that will be used by `Dockerfile` +├── examples <- Examples of PyFunc models implementation ├── test <- Test package ├── pyfuncserver <- Source code of this workflow │ ├── __main__.py <- Entry point of pyfuncserver @@ -120,10 +101,10 @@ Pyfunc server can be configured via following environment variables │ └── rest <- Server implementation for HTTP_JSON protocol │ └── upi <- Server implementation for UPI_V1 protocol ├── .gitignore -├── Makefile <- Makefile +├── Makefile <- Makefile ├── README.md <- The top-level README for developers using this project. ├── requirements.txt <- pyfuncserver dependencies ├── setup.py <- setup.py ├── run.sh <- Script to activate `merlin-model` environment and run pyfuncserver when `docker run` is invoked -``` \ No newline at end of file +``` diff --git a/python/pyfunc-server/docker/local.Dockerfile b/python/pyfunc-server/docker/local.Dockerfile deleted file mode 100644 index ae49e30fa..000000000 --- a/python/pyfunc-server/docker/local.Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2020 The Merlin Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG BASE_IMAGE=ghcr.io/caraml-dev/merlin/merlin-pyfunc-base:0.38.1 -FROM ${BASE_IMAGE} - -# Download and install user model dependencies -ARG MODEL_DEPENDENCIES_URL -COPY ${MODEL_DEPENDENCIES_URL} conda.yaml -RUN conda env create --name merlin-model --file conda.yaml - -# Copy and install pyfunc-server and merlin-sdk dependencies -COPY merlin/python/pyfunc-server /pyfunc-server -COPY merlin/python/sdk /sdk -ENV SDK_PATH=/sdk - -WORKDIR /pyfunc-server -RUN /bin/bash -c ". activate merlin-model && pip uninstall -y merlin-sdk && pip install -r /pyfunc-server/requirements.txt" - -# Download and dry-run user model artifacts and code -ARG MODEL_ARTIFACTS_URL -COPY ${MODEL_ARTIFACTS_URL} model -RUN /bin/bash -c ". activate merlin-model && python -m pyfuncserver --model_dir model --dry_run" - -CMD ["/bin/bash", "/pyfunc-server/run.sh"] diff --git a/python/pyfunc-server/examples/iris/__init__.py b/python/pyfunc-server/examples/iris_http/__init__.py similarity index 100% rename from python/pyfunc-server/examples/iris/__init__.py rename to python/pyfunc-server/examples/iris_http/__init__.py diff --git a/python/pyfunc-server/examples/iris/env.yaml b/python/pyfunc-server/examples/iris_http/env.yaml similarity index 100% rename from python/pyfunc-server/examples/iris/env.yaml rename to python/pyfunc-server/examples/iris_http/env.yaml diff --git a/python/pyfunc-server/examples/iris/iris.py b/python/pyfunc-server/examples/iris_http/iris_http.py similarity index 94% rename from python/pyfunc-server/examples/iris/iris.py rename to python/pyfunc-server/examples/iris_http/iris_http.py index 7cc69ca55..6b87d2f8d 100644 --- a/python/pyfunc-server/examples/iris/iris.py +++ b/python/pyfunc-server/examples/iris_http/iris_http.py @@ -69,7 +69,7 @@ def train_models(): model_instance=IrisModel(), conda_env="env.yaml", artifacts={ - "xgb_model": "models/model_1.bst", - "sklearn_model": "models/model_2.joblib", + "xgb_model": XGB_PATH, + "sklearn_model": SKLEARN_PATH, }, ) diff --git a/python/pyfunc-server/examples/iris/models/model_1.bst b/python/pyfunc-server/examples/iris_http/models/model_1.bst similarity index 100% rename from python/pyfunc-server/examples/iris/models/model_1.bst rename to python/pyfunc-server/examples/iris_http/models/model_1.bst diff --git a/python/pyfunc-server/examples/iris/models/model_2.joblib b/python/pyfunc-server/examples/iris_http/models/model_2.joblib similarity index 93% rename from python/pyfunc-server/examples/iris/models/model_2.joblib rename to python/pyfunc-server/examples/iris_http/models/model_2.joblib index 8c65412b2..0509f7f33 100644 Binary files a/python/pyfunc-server/examples/iris/models/model_2.joblib and b/python/pyfunc-server/examples/iris_http/models/model_2.joblib differ diff --git a/python/sdk/merlin/pyfunc.py b/python/sdk/merlin/pyfunc.py index 2fdfc97ef..24e8467f3 100644 --- a/python/sdk/merlin/pyfunc.py +++ b/python/sdk/merlin/pyfunc.py @@ -6,7 +6,6 @@ import docker import grpc -import mlflow import numpy import pandas from caraml.upi.v1 import upi_pb2 @@ -17,6 +16,8 @@ from merlin.version import VERSION from mlflow.pyfunc import PythonModel +import mlflow + PYFUNC_EXTRA_ARGS_KEY = "__EXTRA_ARGS__" PYFUNC_MODEL_INPUT_KEY = "__INPUT__" PYFUNC_PROTOCOL_KEY = "__PROTOCOL__" @@ -468,7 +469,7 @@ def run_pyfunc_local_server( pyfunc_base_image: str = None, port: int = 8080, env_vars: Dict[str, str] = None, - protocol: Protocol = None, + protocol: Protocol = Protocol.HTTP_JSON, debug: bool = False, ): if pyfunc_base_image is None: @@ -557,7 +558,7 @@ def _run_container( model_full_name, port, env_vars: Dict[str, str] = None, - protocol: Protocol = None, + protocol: Protocol = Protocol.HTTP_JSON, ): docker_client = docker.from_env() @@ -580,6 +581,7 @@ def _run_container( ports = {"8080/tcp": port} if protocol == Protocol.UPI_V1: + env_vars["CARAML_PROTOCOL"] = protocol.value ports = {"9000/tcp": port} try: